[llvm-branch-commits] [llvm] [SCEV] Prove monotonic predicates from loop entry in isKnownViaInduction (PR #231697)
Dmitry Makogon via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 14 04:56:46 PDT 2026
https://github.com/d-makogon updated https://github.com/llvm/llvm-project/pull/231697
>From 788c01c614ba56e2ee495809d7e9bfff45f7c070 Mon Sep 17 00:00:00 2001
From: Dmitrii Makogon <dmakogon at azul.com>
Date: Fri, 9 Oct 2026 17:10:50 +0000
Subject: [PATCH 1/5] [IndVarSimplify][NFC] Add a negative test for zext
elimination
Add @foo3_neg: the loop from @foo3 with a guard on an unrelated value,
so %add3 may be negative and its zext must stay.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
.../IndVarSimplify/iv-widen-elim-ext.ll | 70 +++++++++++++++++++
1 file changed, 70 insertions(+)
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
index a83e9ce74b12ab..45f11263d3e1a1 100644
--- a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
+++ b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
@@ -274,6 +274,76 @@ for.end: ; preds = %for.cond.for.end_cr
ret i32 0
}
+; Same as @foo3, but the guard is on %N, so %add3 may be negative and its zext
+; must stay.
+define i32 @foo3_neg(i32 %M, i32 %N) {
+; CHECK-LABEL: @foo3_neg(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 0, [[N:%.*]]
+; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK: for.body.lr.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[M:%.*]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 1)
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SMAX]] to i64
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_BODY_LR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [100 x i32], ptr @b, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
+; CHECK-NEXT: [[TMP4:%.*]] = trunc nsw i64 [[TMP3]] to i32
+; CHECK-NEXT: [[IDXPROM4:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 [[IDXPROM4]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT: br label [[FOR_INC]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK: for.cond.for.end_crit_edge:
+; CHECK-NEXT: br label [[FOR_END]]
+; CHECK: for.end:
+; CHECK-NEXT: [[CALL:%.*]] = call i32 @dummy(ptr @a, ptr @b)
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ %cmp1 = icmp slt i32 0, %N
+ br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph: ; preds = %entry
+ br label %for.body
+
+for.body: ; preds = %for.body.lr.ph, %for.inc
+ %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+ %idxprom = sext i32 %i.02 to i64
+ %arrayidx = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 %idxprom
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = sext i32 %i.02 to i64
+ %arrayidx2 = getelementptr inbounds [100 x i32], ptr @b, i64 0, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %add = add nsw i32 %0, %1
+ %add3 = add nsw i32 %i.02, %M
+ %idxprom4 = zext i32 %add3 to i64
+ %arrayidx5 = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 %idxprom4
+ store i32 %add, ptr %arrayidx5, align 4
+ br label %for.inc
+
+for.inc: ; preds = %for.body
+ %inc = add nsw i32 %i.02, 1
+ %cmp = icmp slt i32 %inc, %M
+ br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge: ; preds = %for.inc
+ br label %for.end
+
+for.end: ; preds = %for.cond.for.end_crit_edge, %entry
+ %call = call i32 @dummy(ptr @a, ptr @b)
+ ret i32 0
+}
+
%struct.image = type {i32, i32}
define i32 @foo4(ptr %input, i32 %length, ptr %in) {
; CHECK-LABEL: @foo4(
>From 68c56bb87ebb8ad1de7f2353e781bb7744e3aa69 Mon Sep 17 00:00:00 2001
From: Dmitrii Makogon <dmakogon at azul.com>
Date: Fri, 9 Oct 2026 17:11:22 +0000
Subject: [PATCH 2/5] [IndVarSimplify][PhaseOrdering][NFC] Add tests for
widening nsw IV arithmetic
Add tests for for (int i = start; i < N; i++) a[i] = a[i - 32] + 1,
with zext nneg indices and i - 32 nsw only:
- IndVarSimplify: with guards that do and do not prove start
non-negative, whether i - 32 is computed in i64.
- PhaseOrdering (indvars, loop-vectorize): whether a runtime alias check
is emitted.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
.../widen-zext-nneg-add-nsw-runtime-start.ll | 238 ++++++++++++++++++
.../indvars-vectorize-runtime-start.ll | 125 +++++++++
2 files changed, 363 insertions(+)
create mode 100644 llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll b/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
new file mode 100644
index 00000000000000..63551d690150ca
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=indvars -S %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n8:16:32:64-S128"
+
+; for (int i = start; i < 10000; i++) a[i] = a[i - 32] + 1;
+; Array indices are zext nneg, and i - 32 is nsw but not nuw. Where start is
+; known non-negative on loop entry, i - 32 can be computed in i64.
+
+; FIXME: i - 32 should be computed in i64.
+; start and start - 32 are both <u len, and len is non-negative.
+define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
+; CHECK-LABEL: define void @runtime_start(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], ptr [[LEN_P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT: [[LEN_OK:%.*]] = icmp ugt i32 [[LEN]], 9999
+; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]]
+; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32
+; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]]
+; CHECK-NEXT: [[C0:%.*]] = and i1 [[LEN_OK]], [[START_OK]]
+; CHECK-NEXT: [[C1:%.*]] = and i1 [[C0]], [[START_M32_OK]]
+; CHECK-NEXT: [[START_LT:%.*]] = icmp slt i32 [[START]], 10000
+; CHECK-NEXT: [[C:%.*]] = and i1 [[C1]], [[START_LT]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %len = load i32, ptr %len.p, align 4, !range !0
+ %len.ok = icmp ugt i32 %len, 9999
+ %start.ok = icmp ult i32 %start, %len
+ %start.m32 = add i32 %start, -32
+ %start.m32.ok = icmp ult i32 %start.m32, %len
+ %c0 = and i1 %len.ok, %start.ok
+ %c1 = and i1 %c0, %start.m32.ok
+ %start.lt = icmp slt i32 %start, 10000
+ %c = and i1 %c1, %start.lt
+ br i1 %c, label %loop, label %exit
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %idx = add nsw i32 %iv, -32
+ %idx.ext = zext nneg i32 %idx to i64
+ %src = getelementptr inbounds nuw i8, ptr %a, i64 %idx.ext
+ %v = load i8, ptr %src, align 1
+ %v1 = add i8 %v, 1
+ %iv.ext = zext nneg i32 %iv to i64
+ %dst = getelementptr inbounds nuw i8, ptr %a, i64 %iv.ext
+ store i8 %v1, ptr %dst, align 1
+ %iv.next = add nuw nsw i32 %iv, 1
+ %done = icmp sgt i32 %iv, 9998
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; FIXME: i - 32 should be computed in i64.
+; start <u len, and len is non-negative.
+define void @ult_guard_len_range(ptr %a, i32 %start, ptr %len.p) {
+; CHECK-LABEL: define void @ult_guard_len_range(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], ptr [[LEN_P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0]]
+; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[START]], [[LEN]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %len = load i32, ptr %len.p, align 4, !range !0
+ %c = icmp ult i32 %start, %len
+ br i1 %c, label %loop, label %exit
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %idx = add nsw i32 %iv, -32
+ %idx.ext = zext nneg i32 %idx to i64
+ %src = getelementptr inbounds nuw i8, ptr %a, i64 %idx.ext
+ %v = load i8, ptr %src, align 1
+ %v1 = add i8 %v, 1
+ %iv.ext = zext nneg i32 %iv to i64
+ %dst = getelementptr inbounds nuw i8, ptr %a, i64 %iv.ext
+ store i8 %v1, ptr %dst, align 1
+ %iv.next = add nuw nsw i32 %iv, 1
+ %done = icmp sgt i32 %iv, 9998
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; FIXME: i - 32 should be computed in i64.
+; start >=s 32.
+define void @sge_guard(ptr %a, i32 %start) {
+; CHECK-LABEL: define void @sge_guard(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C:%.*]] = icmp sge i32 [[START]], 32
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %c = icmp sge i32 %start, 32
+ br i1 %c, label %loop, label %exit
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %idx = add nsw i32 %iv, -32
+ %idx.ext = zext nneg i32 %idx to i64
+ %src = getelementptr inbounds nuw i8, ptr %a, i64 %idx.ext
+ %v = load i8, ptr %src, align 1
+ %v1 = add i8 %v, 1
+ %iv.ext = zext nneg i32 %iv to i64
+ %dst = getelementptr inbounds nuw i8, ptr %a, i64 %iv.ext
+ store i8 %v1, ptr %dst, align 1
+ %iv.next = add nuw nsw i32 %iv, 1
+ %done = icmp sgt i32 %iv, 9998
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; start <u len does not bound start from below when len may be negative.
+define void @ult_guard_len_unknown(ptr %a, i32 %start, i32 %len) {
+; CHECK-LABEL: define void @ult_guard_len_unknown(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[START]], [[LEN]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %c = icmp ult i32 %start, %len
+ br i1 %c, label %loop, label %exit
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %idx = add nsw i32 %iv, -32
+ %idx.ext = zext nneg i32 %idx to i64
+ %src = getelementptr inbounds nuw i8, ptr %a, i64 %idx.ext
+ %v = load i8, ptr %src, align 1
+ %v1 = add i8 %v, 1
+ %iv.ext = zext nneg i32 %iv to i64
+ %dst = getelementptr inbounds nuw i8, ptr %a, i64 %iv.ext
+ store i8 %v1, ptr %dst, align 1
+ %iv.next = add nuw nsw i32 %iv, 1
+ %done = icmp sgt i32 %iv, 9998
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 2147483647}
+;.
diff --git a/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll b/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
new file mode 100644
index 00000000000000..272ed1e5a3a1a2
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='loop(indvars),loop-vectorize' -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n8:16:32:64-S128"
+
+; for (int i = start; i < 10000; i++) a[i] = a[i - 32] + 1;
+; Array indices are zext nneg, and i - 32 is nsw but not nuw. start is
+; non-negative on loop entry, so the dependence distance between a[i] and
+; a[i - 32] is known and no runtime alias check is needed.
+; FIXME: a runtime alias check is emitted.
+define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
+; CHECK-LABEL: define void @runtime_start(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], ptr [[LEN_P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT: [[LEN_OK:%.*]] = icmp ugt i32 [[LEN]], 9999
+; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]]
+; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32
+; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]]
+; CHECK-NEXT: [[C0:%.*]] = and i1 [[LEN_OK]], [[START_OK]]
+; CHECK-NEXT: [[C1:%.*]] = and i1 [[C0]], [[START_M32_OK]]
+; CHECK-NEXT: [[START_LT:%.*]] = icmp slt i32 [[START]], 10000
+; CHECK-NEXT: [[C:%.*]] = and i1 [[C1]], [[START_LT]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 9999, [[START]]
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 9967, [[START_M32]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[START_M32]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], 15
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP10:%.*]] = and i64 [[TMP3]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = add nuw i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = trunc nuw i64 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = add nsw i32 [[TMP13]], -32
+; CHECK-NEXT: [[TMP15:%.*]] = zext nneg i32 [[TMP14]] to i64
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP15]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP20]], -32
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT: [[V1:%.*]] = add i32 [[V]], 1
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[V1]], ptr [[DST]], align 4
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP21:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP21]], 9998
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %len = load i32, ptr %len.p, align 4, !range !0
+ %len.ok = icmp ugt i32 %len, 9999
+ %start.ok = icmp ult i32 %start, %len
+ %start.m32 = add i32 %start, -32
+ %start.m32.ok = icmp ult i32 %start.m32, %len
+ %c0 = and i1 %len.ok, %start.ok
+ %c1 = and i1 %c0, %start.m32.ok
+ %start.lt = icmp slt i32 %start, 10000
+ %c = and i1 %c1, %start.lt
+ br i1 %c, label %loop, label %exit
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %idx = add nsw i32 %iv, -32
+ %idx.ext = zext nneg i32 %idx to i64
+ %src = getelementptr inbounds nuw i32, ptr %a, i64 %idx.ext
+ %v = load i32, ptr %src, align 4
+ %v1 = add i32 %v, 1
+ %iv.ext = zext nneg i32 %iv to i64
+ %dst = getelementptr inbounds nuw i32, ptr %a, i64 %iv.ext
+ store i32 %v1, ptr %dst, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %done = icmp sgt i32 %iv, 9998
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; [0, INT_MAX)
+!0 = !{i32 0, i32 2147483647}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 2147483647}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]]}
+;.
>From 3307604cbf0766926e14cc2f71c43bb3f1968ce7 Mon Sep 17 00:00:00 2001
From: Dmitrii Makogon <dmakogon at azul.com>
Date: Tue, 13 Oct 2026 21:20:52 +0000
Subject: [PATCH 3/5] [SCEV][NFC] Use isKnownOnEveryIteration in
isKnownViaInduction
isKnownViaInduction and isKnownOnEveryIteration both prove a predicate
on every iteration from loop entry and the backedge, so the next change
can extend one place.
Add a private isKnownOnEveryIteration overload that takes any LHS and
RHS and the loop, and move the entry/backedge check from
isKnownViaInduction into it. The public overload is unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 5 ++
llvm/lib/Analysis/ScalarEvolution.cpp | 54 +++++++++++---------
2 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 17bfec22d65ca7..43c4647140b17a 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2456,6 +2456,11 @@ class ScalarEvolution {
getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
ICmpInst::Predicate Pred);
+ /// Test if the condition described by Pred, LHS, RHS is known to be true on
+ /// every iteration of \p L.
+ bool isKnownOnEveryIteration(CmpPredicate Pred, const SCEV *LHS,
+ const SCEV *RHS, const Loop *L);
+
/// Return SCEV no-wrap flags that can be proven based on reasoning about
/// how poison produced from no-wrap flags on this value (e.g. a nuw add)
/// would trigger undefined behavior on overflow.
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index fef9529ee0f9e4..bc1829c78d3f21 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11387,30 +11387,7 @@ bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
return DT.properlyDominates(L1->getHeader(), L2->getHeader());
});
- // Get init and post increment value for LHS.
- auto SplitLHS = SplitIntoInitAndPostInc(MDL, LHS);
- // if LHS contains unknown non-invariant SCEV then bail out.
- if (SplitLHS.first == getCouldNotCompute())
- return false;
- assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
- // Get init and post increment value for RHS.
- auto SplitRHS = SplitIntoInitAndPostInc(MDL, RHS);
- // if RHS contains unknown non-invariant SCEV then bail out.
- if (SplitRHS.first == getCouldNotCompute())
- return false;
- assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
- // It is possible that init SCEV contains an invariant load but it does
- // not dominate MDL and is not available at MDL loop entry, so we should
- // check it here.
- if (!isAvailableAtLoopEntry(SplitLHS.first, MDL) ||
- !isAvailableAtLoopEntry(SplitRHS.first, MDL))
- return false;
-
- // It seems backedge guard check is faster than entry one so in some cases
- // it can speed up whole estimation by short circuit
- return isLoopBackedgeGuardedByCond(MDL, Pred, SplitLHS.second,
- SplitRHS.second) &&
- isLoopEntryGuardedByCond(MDL, Pred, SplitLHS.first, SplitRHS.first);
+ return isKnownOnEveryIteration(Pred, LHS, RHS, MDL);
}
bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
@@ -11464,6 +11441,35 @@ bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
isLoopBackedgeGuardedByCond(L, Pred, LHS->getPostIncExpr(*this), RHS);
}
+bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
+ const SCEV *LHS, const SCEV *RHS,
+ const Loop *L) {
+ // Get init and post increment value for LHS.
+ auto SplitLHS = SplitIntoInitAndPostInc(L, LHS);
+ // if LHS contains unknown non-invariant SCEV then bail out.
+ if (SplitLHS.first == getCouldNotCompute())
+ return false;
+ assert(SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
+ // Get init and post increment value for RHS.
+ auto SplitRHS = SplitIntoInitAndPostInc(L, RHS);
+ // if RHS contains unknown non-invariant SCEV then bail out.
+ if (SplitRHS.first == getCouldNotCompute())
+ return false;
+ assert(SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
+ // It is possible that init SCEV contains an invariant load but it does
+ // not dominate L and is not available at L loop entry, so we should
+ // check it here.
+ if (!isAvailableAtLoopEntry(SplitLHS.first, L) ||
+ !isAvailableAtLoopEntry(SplitRHS.first, L))
+ return false;
+
+ // It seems backedge guard check is faster than entry one so in some cases
+ // it can speed up whole estimation by short circuit
+ return isLoopBackedgeGuardedByCond(L, Pred, SplitLHS.second,
+ SplitRHS.second) &&
+ isLoopEntryGuardedByCond(L, Pred, SplitLHS.first, SplitRHS.first);
+}
+
std::optional<ScalarEvolution::MonotonicPredicateType>
ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
ICmpInst::Predicate Pred) {
>From 31501968f6fad5efce4a197286734d75d970a5b5 Mon Sep 17 00:00:00 2001
From: Dmitrii Makogon <dmakogon at azul.com>
Date: Tue, 13 Oct 2026 22:37:30 +0000
Subject: [PATCH 4/5] [SCEV][NFC] Factor getMonotonicPredicate out of
getLoopInvariantPredicate
The next change needs the same setup as getLoopInvariantPredicate.
Move it into a private helper: put the loop-invariant operand on the
RHS, check that the other side is a recurrence of the loop, and compute
the predicate's monotonicity.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 19 +++++++++++
llvm/lib/Analysis/ScalarEvolution.cpp | 33 ++++++++++++++------
2 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 43c4647140b17a..c9ed3948b6fd3a 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -2461,6 +2461,25 @@ class ScalarEvolution {
bool isKnownOnEveryIteration(CmpPredicate Pred, const SCEV *LHS,
const SCEV *RHS, const Loop *L);
+ /// "AR Pred RHS", with AR a recurrence of a loop and RHS invariant in it,
+ /// and how the predicate changes as the loop iterates.
+ struct MonotonicPredicate {
+ CmpPredicate Pred;
+ const SCEVAddRecExpr *AR;
+ const SCEV *RHS;
+ MonotonicPredicateType Type;
+ /// Whether the operands were swapped to put the recurrence on the left.
+ bool Swapped;
+ };
+
+ /// If one of LHS and RHS is a recurrence of \p L and the other is invariant
+ /// in \p L, and "LHS Pred RHS" is monotonic in \p L, return it with the
+ /// recurrence on the left. Otherwise, return std::nullopt.
+ std::optional<MonotonicPredicate> getMonotonicPredicate(CmpPredicate Pred,
+ const SCEV *LHS,
+ const SCEV *RHS,
+ const Loop *L);
+
/// Return SCEV no-wrap flags that can be proven based on reasoning about
/// how poison produced from no-wrap flags on this value (e.g. a nuw add)
/// would trigger undefined behavior on overflow.
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index bc1829c78d3f21..58f10f6b290c83 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11533,26 +11533,41 @@ ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
return std::nullopt;
}
-std::optional<ScalarEvolution::LoopInvariantPredicate>
-ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
- const SCEV *RHS, const Loop *L,
- const Instruction *CtxI) {
+std::optional<ScalarEvolution::MonotonicPredicate>
+ScalarEvolution::getMonotonicPredicate(CmpPredicate Pred, const SCEV *LHS,
+ const SCEV *RHS, const Loop *L) {
// If there is a loop-invariant, force it into the RHS, otherwise bail out.
+ bool Swapped = false;
if (!isLoopInvariant(RHS, L)) {
if (!isLoopInvariant(LHS, L))
return std::nullopt;
std::swap(LHS, RHS);
Pred = ICmpInst::getSwappedCmpPredicate(Pred);
+ Swapped = true;
}
- const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS);
- if (!ArLHS || ArLHS->getLoop() != L)
+ const auto *AR = dyn_cast<SCEVAddRecExpr>(LHS);
+ if (!AR || AR->getLoop() != L)
return std::nullopt;
- auto MonotonicType = getMonotonicPredicateType(ArLHS, Pred);
- if (!MonotonicType)
+ auto Type = getMonotonicPredicateType(AR, Pred);
+ if (!Type)
+ return std::nullopt;
+ return MonotonicPredicate{Pred, AR, RHS, *Type, Swapped};
+}
+
+std::optional<ScalarEvolution::LoopInvariantPredicate>
+ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
+ const SCEV *RHS, const Loop *L,
+ const Instruction *CtxI) {
+ auto MP = getMonotonicPredicate(Pred, LHS, RHS, L);
+ if (!MP)
return std::nullopt;
+ Pred = MP->Pred;
+ const SCEVAddRecExpr *ArLHS = MP->AR;
+ LHS = ArLHS;
+ RHS = MP->RHS;
// If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
// true as the loop iterates, and the backedge is control dependent on
// "ArLHS `Pred` RHS" == true then we can reason as follows:
@@ -11570,7 +11585,7 @@ ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
//
// A similar reasoning applies for a monotonically decreasing predicate, by
// replacing true with false and false with true in the above two bullets.
- bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
+ bool Increasing = MP->Type == ScalarEvolution::MonotonicallyIncreasing;
auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
if (isLoopBackedgeGuardedByCond(L, P, LHS, RHS))
>From a7ac0b02c63025e3ec87b8d01685695242692f0f Mon Sep 17 00:00:00 2001
From: Dmitrii Makogon <dmakogon at azul.com>
Date: Tue, 13 Oct 2026 22:30:55 +0000
Subject: [PATCH 5/5] [SCEV] Prove monotonic predicates from loop entry in
isKnownViaInduction
Prove "IV pred Inv" on every iteration of a loop when Inv is
loop-invariant, the predicate holds on loop entry, and it can only change
from false to true as the loop iterates.
Before, the predicate also had to be proven on the backedge, which
usually fails when the loop exit only bounds the IV from above. E.g.
i >=s 0 for an nsw IV starting at s was not proven even when s >=s 0 was
known on entry. As a result, in a loop guarded by "start <u len",
IndVars could not compute a[i - 32] in i64.
Co-Authored-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++-
.../IndVarSimplify/eliminate-comparison.ll | 65 ++++++++++++--
.../IndVarSimplify/iv-widen-elim-ext.ll | 6 +-
.../monotonic-predicate-samesign.ll | 78 +++++++++++++++++
.../IndVarSimplify/trivial-guard.ll | 6 +-
.../widen-zext-nneg-add-nsw-runtime-start.ll | 33 +++----
.../indvars-vectorize-runtime-start.ll | 32 ++-----
.../Analysis/ScalarEvolutionTest.cpp | 85 +++++++++++++++++++
8 files changed, 259 insertions(+), 61 deletions(-)
create mode 100644 llvm/test/Transforms/IndVarSimplify/monotonic-predicate-samesign.ll
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 58f10f6b290c83..76cbb3217d48bf 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11465,9 +11465,18 @@ bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
// It seems backedge guard check is faster than entry one so in some cases
// it can speed up whole estimation by short circuit
- return isLoopBackedgeGuardedByCond(L, Pred, SplitLHS.second,
- SplitRHS.second) &&
- isLoopEntryGuardedByCond(L, Pred, SplitLHS.first, SplitRHS.first);
+ if (isLoopBackedgeGuardedByCond(L, Pred, SplitLHS.second, SplitRHS.second))
+ return isLoopEntryGuardedByCond(L, Pred, SplitLHS.first, SplitRHS.first);
+
+ // If one side is a recurrence of L and the other side is invariant in L,
+ // and the predicate can only change from false to true as the loop
+ // iterates, it holds on every iteration once it holds on entry.
+ auto MP = getMonotonicPredicate(Pred.dropSameSign(), LHS, RHS, L);
+ if (!MP || MP->Type != MonotonicallyIncreasing)
+ return false;
+ if (MP->Swapped)
+ std::swap(SplitLHS, SplitRHS);
+ return isLoopEntryGuardedByCond(L, MP->Pred, SplitLHS.first, SplitRHS.first);
}
std::optional<ScalarEvolution::MonotonicPredicateType>
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
index 08f9856ac603d6..853de678655ad9 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
@@ -569,6 +569,8 @@ leave:
ret void
}
+; The entry condition implies %len is negative, so the loop exits on its first
+; iteration.
define void @func_17(ptr %len.ptr) {
; CHECK-LABEL: @func_17(
; CHECK-NEXT: entry:
@@ -579,18 +581,13 @@ define void @func_17(ptr %len.ptr) {
; CHECK-NEXT: [[ENTRY_COND:%.*]] = and i1 [[ENTRY_COND_0]], [[ENTRY_COND_1]]
; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
; CHECK: loop.preheader:
-; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[LEN]], i32 0)
-; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[SMAX]], -5
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_INC:%.*]], [[BE:%.*]] ], [ -6, [[LOOP_PREHEADER]] ]
; CHECK-NEXT: call void @side_effect()
-; CHECK-NEXT: [[IV_INC]] = add nsw i32 [[IV]], 1
-; CHECK-NEXT: br i1 true, label [[BE]], label [[LEAVE_LOOPEXIT:%.*]]
+; CHECK-NEXT: br i1 true, label [[BE:%.*]], label [[LEAVE_LOOPEXIT:%.*]]
; CHECK: be:
; CHECK-NEXT: call void @side_effect()
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_INC]], [[TMP0]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LEAVE_LOOPEXIT]]
+; CHECK-NEXT: br i1 false, label [[LOOP]], label [[LEAVE_LOOPEXIT]]
; CHECK: leave.loopexit:
; CHECK-NEXT: br label [[LEAVE]]
; CHECK: leave:
@@ -611,9 +608,59 @@ loop:
%iv.inc = add i32 %iv, 1
%iv.2.inc = add i32 %iv.2, 1
%iv.cmp = icmp slt i32 %iv, %len.add.5
+ br i1 %iv.cmp, label %be, label %leave
+
+be:
+ call void @side_effect()
+ %be.cond = icmp slt i32 %iv.2, %len
+ br i1 %be.cond, label %loop, label %leave
+
+leave:
+ ret void
+}
+
+; %iv + 1 s< %len.add.5 follows from %iv.2 s< %len, since %len s< INT_MIN - (-5)
+; from the entry condition. Unlike @func_17, the loop can iterate.
+define void @func_17_iterating(ptr %len.ptr) {
+; CHECK-LABEL: @func_17_iterating(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_PTR:%.*]], align 4
+; CHECK-NEXT: [[LEN_ADD_5:%.*]] = add i32 [[LEN]], -5
+; CHECK-NEXT: [[ENTRY_COND_0:%.*]] = icmp slt i32 [[LEN]], -2147483643
+; CHECK-NEXT: [[ENTRY_COND_1:%.*]] = icmp slt i32 2147483642, [[LEN_ADD_5]]
+; CHECK-NEXT: [[ENTRY_COND:%.*]] = and i1 [[ENTRY_COND_0]], [[ENTRY_COND_1]]
+; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]]
+; CHECK: loop.preheader:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_INC:%.*]], [[BE:%.*]] ], [ 2147483642, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT: call void @side_effect()
+; CHECK-NEXT: [[IV_INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: br i1 true, label [[BE]], label [[LEAVE_LOOPEXIT:%.*]]
+; CHECK: be:
+; CHECK-NEXT: call void @side_effect()
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_INC]], [[LEN_ADD_5]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LEAVE_LOOPEXIT]]
+; CHECK: leave.loopexit:
+; CHECK-NEXT: br label [[LEAVE]]
+; CHECK: leave:
+; CHECK-NEXT: ret void
+;
+entry:
+ %len = load i32, ptr %len.ptr
+ %len.add.5 = add i32 %len, -5
+ %entry.cond.0 = icmp slt i32 %len, 2147483653 ;; 2147483653 == INT_MIN - (-5)
+ %entry.cond.1 = icmp slt i32 2147483642, %len.add.5
+ %entry.cond = and i1 %entry.cond.0, %entry.cond.1
+ br i1 %entry.cond, label %loop, label %leave
-; Deduces {-5,+,1} s< (-5 + %len) from {0,+,1} < %len
-; since %len s< INT_MIN - (-5) from the entry condition
+loop:
+ %iv.2 = phi i32 [ -2147483648, %entry ], [ %iv.2.inc, %be ]
+ %iv = phi i32 [ 2147483642, %entry ], [ %iv.inc, %be ]
+ call void @side_effect()
+ %iv.inc = add i32 %iv, 1
+ %iv.2.inc = add i32 %iv.2, 1
+ %iv.cmp = icmp slt i32 %iv, %len.add.5
br i1 %iv.cmp, label %be, label %leave
be:
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
index 45f11263d3e1a1..4ce57753da6731 100644
--- a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
+++ b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
@@ -206,7 +206,7 @@ for.end: ; preds = %for.cond.for.end_cr
declare i32 @dummy(ptr, ptr)
-; A case where zext should not be eliminated when its operands could only be extended by sext.
+; The guard 0 <s %M makes %add3 non-negative, so its zext is eliminated.
define i32 @foo3(i32 %M) {
; CHECK-LABEL: @foo3(
; CHECK-NEXT: entry:
@@ -224,9 +224,7 @@ define i32 @foo3(i32 %M) {
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT: [[TMP4:%.*]] = trunc nsw i64 [[TMP3]] to i32
-; CHECK-NEXT: [[IDXPROM4:%.*]] = zext i32 [[TMP4]] to i64
-; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 [[IDXPROM4]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 [[TMP3]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: br label [[FOR_INC]]
; CHECK: for.inc:
diff --git a/llvm/test/Transforms/IndVarSimplify/monotonic-predicate-samesign.ll b/llvm/test/Transforms/IndVarSimplify/monotonic-predicate-samesign.ll
new file mode 100644
index 00000000000000..67c3d8a87f6da3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/monotonic-predicate-samesign.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=indvars -S < %s | FileCheck %s
+
+; s >s inv does not make %c true on every iteration: with s = 3 and inv = -5,
+; %c is false on the last iteration, where %iv = INT_MIN.
+define i32 @samesign_ugt_signed_guard(i32 %s, i32 %inv) {
+; CHECK-LABEL: define i32 @samesign_ugt_signed_guard(
+; CHECK-SAME: i32 [[S:%.*]], i32 [[INV:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[G:%.*]] = icmp sgt i32 [[S]], [[INV]]
+; CHECK-NEXT: br i1 [[G]], label %[[LOOP_PREHEADER:.*]], label %[[BAIL:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[S]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp samesign ugt i32 [[IV]], [[INV]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], -2147483648
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[C_LCSSA:%.*]] = phi i1 [ [[C]], %[[LOOP]] ]
+; CHECK-NEXT: [[R:%.*]] = zext i1 [[C_LCSSA]] to i32
+; CHECK-NEXT: ret i32 [[R]]
+; CHECK: [[BAIL]]:
+; CHECK-NEXT: ret i32 7
+;
+entry:
+ %g = icmp sgt i32 %s, %inv
+ br i1 %g, label %loop, label %bail
+loop:
+ %iv = phi i32 [ %s, %entry ], [ %iv.next, %loop ]
+ %c = icmp samesign ugt i32 %iv, %inv
+ %iv.next = add nuw i32 %iv, 1
+ %ec = icmp eq i32 %iv, u0x80000000
+ br i1 %ec, label %exit, label %loop
+exit:
+ %r = zext i1 %c to i32
+ ret i32 %r
+bail:
+ ret i32 7
+}
+
+; s >u inv is known on entry and the IV is nuw, so %c folds to true.
+define i32 @samesign_ugt_unsigned_guard(i32 %s, i32 %inv) {
+; CHECK-LABEL: define i32 @samesign_ugt_unsigned_guard(
+; CHECK-SAME: i32 [[S:%.*]], i32 [[INV:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[G:%.*]] = icmp ugt i32 [[S]], [[INV]]
+; CHECK-NEXT: br i1 [[G]], label %[[LOOP_PREHEADER:.*]], label %[[BAIL:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[S]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], -2147483648
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[C_LCSSA:%.*]] = phi i1 [ true, %[[LOOP]] ]
+; CHECK-NEXT: [[R:%.*]] = zext i1 [[C_LCSSA]] to i32
+; CHECK-NEXT: ret i32 [[R]]
+; CHECK: [[BAIL]]:
+; CHECK-NEXT: ret i32 7
+;
+entry:
+ %g = icmp ugt i32 %s, %inv
+ br i1 %g, label %loop, label %bail
+loop:
+ %iv = phi i32 [ %s, %entry ], [ %iv.next, %loop ]
+ %c = icmp samesign ugt i32 %iv, %inv
+ %iv.next = add nuw i32 %iv, 1
+ %ec = icmp eq i32 %iv, u0x80000000
+ br i1 %ec, label %exit, label %loop
+exit:
+ %r = zext i1 %c to i32
+ ret i32 %r
+bail:
+ ret i32 7
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll
index c1c9a90d9831dc..e29e62eb82c145 100644
--- a/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll
+++ b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll
@@ -3,6 +3,7 @@
declare i1 @cond()
+; One of the two loops is unreachable, so its check may fold either way.
define void @test_01(i32 %x) {
; CHECK-LABEL: @test_01(
; CHECK-NEXT: entry:
@@ -20,7 +21,7 @@ define void @test_01(i32 %x) {
; CHECK-NEXT: [[LOOP_COND_1:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]]
; CHECK: loop.2:
-; CHECK-NEXT: br i1 true, label [[GUARDED_2:%.*]], label [[FAIL_LOOPEXIT1:%.*]]
+; CHECK-NEXT: br i1 false, label [[GUARDED_2:%.*]], label [[FAIL_LOOPEXIT1:%.*]]
; CHECK: guarded.2:
; CHECK-NEXT: [[LOOP_COND_2:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[LOOP_COND_2]], label [[LOOP_2]], label [[EXIT_LOOPEXIT2:%.*]]
@@ -67,6 +68,7 @@ fail:
unreachable
}
+; Same as @test_01, with the other loop unreachable.
define void @test_02(i32 %x) {
; CHECK-LABEL: @test_02(
; CHECK-NEXT: entry:
@@ -76,7 +78,7 @@ define void @test_02(i32 %x) {
; CHECK: loop.1.preheader:
; CHECK-NEXT: br label [[LOOP_1:%.*]]
; CHECK: loop.1:
-; CHECK-NEXT: br i1 true, label [[GUARDED_1:%.*]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK-NEXT: br i1 false, label [[GUARDED_1:%.*]], label [[FAIL_LOOPEXIT:%.*]]
; CHECK: guarded.1:
; CHECK-NEXT: [[LOOP_COND_1:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll b/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
index 63551d690150ca..7e58e0eddb9ebe 100644
--- a/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
+++ b/llvm/test/Transforms/IndVarSimplify/widen-zext-nneg-add-nsw-runtime-start.ll
@@ -5,9 +5,8 @@ target datalayout = "e-m:e-i64:64-n8:16:32:64-S128"
; for (int i = start; i < 10000; i++) a[i] = a[i - 32] + 1;
; Array indices are zext nneg, and i - 32 is nsw but not nuw. Where start is
-; known non-negative on loop entry, i - 32 can be computed in i64.
+; known non-negative on loop entry, i - 32 is computed in i64.
-; FIXME: i - 32 should be computed in i64.
; start and start - 32 are both <u len, and len is non-negative.
define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK-LABEL: define void @runtime_start(
@@ -28,17 +27,15 @@ define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
-; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[IDX_EXT:%.*]] = add nsw i64 [[INDVARS_IV]], -32
; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 10000
; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
@@ -75,7 +72,6 @@ exit:
ret void
}
-; FIXME: i - 32 should be computed in i64.
; start <u len, and len is non-negative.
define void @ult_guard_len_range(ptr %a, i32 %start, ptr %len.p) {
; CHECK-LABEL: define void @ult_guard_len_range(
@@ -86,20 +82,20 @@ define void @ult_guard_len_range(ptr %a, i32 %start, ptr %len.p) {
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[START]], i32 9999)
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[SMAX]], 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
-; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[IDX_EXT:%.*]] = add nsw i64 [[INDVARS_IV]], -32
; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[TMP1]]
; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
@@ -129,7 +125,6 @@ exit:
ret void
}
-; FIXME: i - 32 should be computed in i64.
; start >=s 32.
define void @sge_guard(ptr %a, i32 %start) {
; CHECK-LABEL: define void @sge_guard(
@@ -139,20 +134,20 @@ define void @sge_guard(ptr %a, i32 %start) {
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[START]], i32 9999)
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[SMAX]], 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP1]], -32
-; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[IDX_EXT:%.*]] = add nsw i64 [[INDVARS_IV]], -32
; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC]], align 1
; CHECK-NEXT: [[V1:%.*]] = add i8 [[V]], 1
; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i8 [[V1]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP2]], 9998
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[TMP1]]
; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
diff --git a/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll b/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
index 272ed1e5a3a1a2..24237265a30032 100644
--- a/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
+++ b/llvm/test/Transforms/PhaseOrdering/indvars-vectorize-runtime-start.ll
@@ -7,7 +7,6 @@ target datalayout = "e-m:e-i64:64-n8:16:32:64-S128"
; Array indices are zext nneg, and i - 32 is nsw but not nuw. start is
; non-negative on loop entry, so the dependence distance between a[i] and
; a[i - 32] is known and no runtime alias check is needed.
-; FIXME: a runtime alias check is emitted.
define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK-LABEL: define void @runtime_start(
; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], ptr [[LEN_P:%.*]]) {
@@ -28,18 +27,7 @@ define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 9967, [[START_M32]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[START_M32]] to i64
-; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
-; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[TMP5]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP9]], 15
-; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP10:%.*]] = and i64 [[TMP3]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP10]]
@@ -48,9 +36,7 @@ define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = add nuw i64 [[TMP0]], [[INDEX]]
-; CHECK-NEXT: [[TMP13:%.*]] = trunc nuw i64 [[TMP12]] to i32
-; CHECK-NEXT: [[TMP14:%.*]] = add nsw i32 [[TMP13]], -32
-; CHECK-NEXT: [[TMP15:%.*]] = zext nneg i32 [[TMP14]] to i64
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw i64 [[TMP12]], -32
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP15]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP17:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
@@ -63,22 +49,20 @@ define void @runtime_start(ptr %a, i32 %start, ptr %len.p) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[LOOP_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[LOOP_PREHEADER]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[IDX:%.*]] = add nsw i32 [[TMP20]], -32
-; CHECK-NEXT: [[IDX_EXT:%.*]] = zext nneg i32 [[IDX]] to i64
+; CHECK-NEXT: [[IDX_EXT:%.*]] = add nsw i64 [[INDVARS_IV]], -32
; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-NEXT: [[V1:%.*]] = add i32 [[V]], 1
; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i32 [[V1]], ptr [[DST]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP21:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT: [[DONE:%.*]] = icmp sgt i32 [[TMP21]], 9998
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 10000
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -121,5 +105,5 @@ exit:
; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META2]]}
;.
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index c7a3f97389900d..925924897d86c0 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -74,6 +74,12 @@ static std::optional<APInt> computeConstantDifference(ScalarEvolution &SE,
const SCEV *FoundRHS) {
return SE.isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS);
}
+
+ static bool isKnownOnEveryIteration(ScalarEvolution &SE, CmpPredicate Pred,
+ const SCEV *LHS, const SCEV *RHS,
+ const Loop *L) {
+ return SE.isKnownOnEveryIteration(Pred, LHS, RHS, L);
+ }
};
TEST_F(ScalarEvolutionsTest, SCEVUnknownRAUW) {
@@ -1649,6 +1655,85 @@ TEST_F(ScalarEvolutionsTest, ImpliedNonNegativeFromULT) {
});
}
+// A predicate that can only change from false to true as the loop iterates
+// holds on every iteration if it holds on loop entry.
+TEST_F(ScalarEvolutionsTest, KnownOnEveryIterationMonotonic) {
+ LLVMContext C;
+ SMDiagnostic Err;
+ std::unique_ptr<Module> M = parseAssemblyString(
+ "define void @nsw(i32 %s, i32 %n, i32 %inv) { "
+ "entry: "
+ " %g = icmp sge i32 %s, 0 "
+ " br i1 %g, label %loop, label %exit "
+ "loop: "
+ " %iv = phi i32 [ %s, %entry ], [ %iv.next, %loop ] "
+ " %iv.next = add nsw i32 %iv, 1 "
+ " %var = add i32 %inv, %iv "
+ " %ec = icmp slt i32 %iv.next, %n "
+ " br i1 %ec, label %loop, label %exit "
+ "exit: "
+ " ret void "
+ "} "
+ "define void @nest(i32 %s, i32 %n) { "
+ "entry: "
+ " %g = icmp sge i32 %s, 0 "
+ " br i1 %g, label %outer, label %exit "
+ "outer: "
+ " %o = phi i32 [ 0, %entry ], [ %o.next, %outer.latch ] "
+ " %og = icmp sge i32 %s, %o "
+ " br i1 %og, label %inner, label %outer.latch "
+ "inner: "
+ " %i = phi i32 [ %s, %outer ], [ %i.next, %inner ] "
+ " %i.next = add nsw i32 %i, 1 "
+ " %iec = icmp slt i32 %i.next, %n "
+ " br i1 %iec, label %inner, label %outer.latch "
+ "outer.latch: "
+ " %o.next = add nsw i32 %o, 1 "
+ " %oec = icmp slt i32 %o.next, %n "
+ " br i1 %oec, label %outer, label %exit "
+ "exit: "
+ " ret void "
+ "} ",
+ Err, C);
+
+ ASSERT_TRUE(M && "Could not parse module?");
+ ASSERT_TRUE(!verifyModule(*M) && "Must have been well formed!");
+
+ const auto SGE = ICmpInst::ICMP_SGE, SLE = ICmpInst::ICMP_SLE;
+
+ runWithSE(*M, "nsw", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+ Instruction *IVInst = getInstructionByName(F, "iv");
+ const SCEV *IV = SE.getSCEV(IVInst);
+ const Loop *L = LI.getLoopFor(IVInst->getParent());
+ const SCEV *Zero = SE.getZero(IV->getType());
+ EXPECT_TRUE(isKnownOnEveryIteration(SE, SGE, IV, Zero, L));
+ EXPECT_TRUE(isKnownOnEveryIteration(SE, SLE, Zero, IV, L));
+ EXPECT_TRUE(SE.isKnownPredicate(SGE, IV, Zero));
+ // Decreasing.
+ EXPECT_FALSE(
+ isKnownOnEveryIteration(SE, SLE, IV, SE.getMinusOne(IV->getType()), L));
+ // Not loop-invariant.
+ EXPECT_FALSE(isKnownOnEveryIteration(
+ SE, SGE, IV, SE.getSCEV(getInstructionByName(F, "var")), L));
+ // Not known on entry.
+ EXPECT_FALSE(
+ isKnownOnEveryIteration(SE, SGE, IV, SE.getSCEV(F.getArg(2)), L));
+ });
+
+ runWithSE(*M, "nest", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+ const SCEV *I = SE.getSCEV(getInstructionByName(F, "i"));
+ const SCEV *O = SE.getSCEV(getInstructionByName(F, "o"));
+ const Loop *Inner =
+ LI.getLoopFor(getInstructionByName(F, "i")->getParent());
+ const Loop *Outer =
+ LI.getLoopFor(getInstructionByName(F, "o")->getParent());
+ // The outer recurrence is invariant in the inner loop.
+ EXPECT_TRUE(isKnownOnEveryIteration(SE, SGE, I, O, Inner));
+ EXPECT_TRUE(isKnownOnEveryIteration(SE, SLE, O, I, Inner));
+ EXPECT_FALSE(isKnownOnEveryIteration(SE, SLE, O, I, Outer));
+ });
+}
+
TEST_F(ScalarEvolutionsTest, MatchURem) {
LLVMContext C;
SMDiagnostic Err;
More information about the llvm-branch-commits
mailing list