[llvm] [LV] Add additional SCEV expansion tests with AddRecs. (NFC) (PR #210159)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 03:42:53 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/210159
>From f2360360e495439d528d912c8eadd009a5939b7a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 16 Jul 2026 19:23:14 +0100
Subject: [PATCH 1/2] [LV] Add additional SCEV expansion tests with AddRecs.
(NFC)
Add additional test coverage for SCEV expansion of AddRecs, including
cases where extends are needed, different start and steps, as well as
cases where the outer header is not the plan's entry block.
---
.../LoopVectorize/VPlan/expand-scev.ll | 206 ++++++++++++++++++
.../nested-loops-scev-expansion.ll | 81 +++++++
2 files changed, 287 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
index 1e55658ce19ae..45543a78a27cd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
@@ -357,6 +357,212 @@ exit:
ret void
}
+define void @addrec_outer_iv_narrow(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_outer_iv_narrow'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%0> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %indvar = phi i64 [ %indvar.next, %outer.latch ], [ 0, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = add i64 %indvar, 5
+; CHECK-NEXT: IR %ext = zext i32 %outer.iv to i64
+; CHECK-NEXT: Successor(s): vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ext = zext i32 %outer.iv to i64
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 1
+ %bound = add i64 %ext, 5
+ %cmp.inner = icmp ult i64 %iv.next, %bound
+ br i1 %cmp.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i32 %outer.iv, 1
+ %cmp.outer = icmp ult i32 %outer.iv.next, 100
+ br i1 %cmp.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; {0,+,2}<nuw> from outer loop needs expansion.
+define void @addrec_non_unit_outer_stride(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_unit_outer_stride'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = add i64 %outer.iv, 4
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %outer.iv, 5
+ %cmp.inner = icmp ult i64 %iv.next, %bound
+ br i1 %cmp.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 2
+ %cmp.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %cmp.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; AddRec in outer-most loop needs expansion.
+define void @addrec_over_grandparent_loop(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_over_grandparent_loop'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle>:
+; CHECK-NEXT: IR %mid = phi i64 [ 0, %outermost ], [ %mid.next, %mid.latch ]
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outermost
+
+outermost:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.iv.latch ]
+ br label %middle
+
+middle:
+ %mid = phi i64 [ 0, %outermost ], [ %mid.next, %mid.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %middle ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %outer.iv, 5
+ %cmp.inner = icmp ult i64 %iv.next, %bound
+ br i1 %cmp.inner, label %inner, label %mid.latch
+
+mid.latch:
+ %mid.next = add nuw i64 %mid, 1
+ %cm = icmp ult i64 %mid.next, 50
+ br i1 %cm, label %middle, label %outer.iv.latch
+
+outer.iv.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %co = icmp ult i64 %outer.iv.next, 100
+ br i1 %co, label %outermost, label %exit
+
+exit:
+ ret void
+}
+
+define void @addrec_phi_not_in_inner_preheader(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_phi_not_in_inner_preheader'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<inner.ph>:
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %scaled = add i64 %outer.iv, 5
+ %cc = icmp ult i64 %outer.iv, 1000
+ br i1 %cc, label %inner.ph, label %exit
+
+inner.ph:
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %inner.ph ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %cmp.inner = icmp ult i64 %iv.next, %scaled
+ br i1 %cmp.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ br label %outer.header
+
+exit:
+ ret void
+}
+
+; {4,+,4}<nuw> from outer loop needs expansion.
+define void @addrec_nuw_flags(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_nuw_flags'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%3> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = shl nuw nsw i64 %outer.iv, 2
+; CHECK-NEXT: IR %1 = add i64 %0, 4
+; CHECK-NEXT: IR %2 = udiv i64 %1, 3
+; CHECK-NEXT: IR %3 = add nuw nsw i64 %2, 1
+; CHECK-NEXT: IR %m = mul nuw i64 %outer.iv, 4
+; CHECK-NEXT: IR %bound = add nuw i64 %m, 5
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%3>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %m = mul nuw i64 %outer.iv, 4
+ %bound = add nuw i64 %m, 5
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %cmp.inner = icmp ult i64 %iv.next, %bound
+ br i1 %cmp.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %cmp.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %cmp.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
index 491882b796ef1..0a8e4fccb715f 100644
--- a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
@@ -404,4 +404,85 @@ exit:
ret void
}
+; SCEV expander creates a new canonical IV in the outer loop.
+define void @test_expand_new_canonical_iv_non_zero_start(ptr %dst) {
+; CHECK-LABEL: define void @test_expand_new_canonical_iv_non_zero_start(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER:.*]]
+; CHECK: [[OUTER]]:
+; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDVAR]], 17
+; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[BOUND:%.*]] = add i64 [[OUTER_IV]], 8
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 3
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], 6
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 9
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT: store i8 0, ptr [[TMP8]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP9]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP10]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP3]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 3
+; CHECK-NEXT: [[CMP_INNER:%.*]] = icmp ult i64 [[IV_NEXT]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP_INNER]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[CMP_OUTER:%.*]] = icmp ult i64 [[OUTER_IV_NEXT]], 110
+; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT: br i1 [[CMP_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 10, %entry ], [ %outer.iv.next, %outer.latch ]
+ %bound = add i64 %outer.iv, 8
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %cmp.inner = icmp ult i64 %iv.next, %bound
+ br i1 %cmp.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %cmp.outer = icmp ult i64 %outer.iv.next, 110
+ br i1 %cmp.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
declare void @cond()
>From 82a5f612aa20c32b80c7859358e95c6d23e9f7d8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jul 2026 11:27:57 +0100
Subject: [PATCH 2/2] !fixup add non-affine addrec tests, address comments
---
.../LoopVectorize/VPlan/expand-scev.ll | 144 +++++++++++++++---
.../nested-loops-scev-expansion.ll | 24 +--
2 files changed, 131 insertions(+), 37 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
index 45543a78a27cd..89279ea5629dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
@@ -345,13 +345,13 @@ inner:
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
%bound = add i64 %outer.iv, 5
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i64 %outer.iv, 1
- %cmp.outer = icmp ult i64 %outer.iv.next, 100
- br i1 %cmp.outer, label %outer, label %exit
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
@@ -383,19 +383,20 @@ inner:
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 1
%bound = add i64 %ext, 5
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i32 %outer.iv, 1
- %cmp.outer = icmp ult i32 %outer.iv.next, 100
- br i1 %cmp.outer, label %outer, label %exit
+ %ec.outer = icmp ult i32 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
}
-; {0,+,2}<nuw> from outer loop needs expansion.
+; AddRec {0,+,2}<nuw> from the outer loop feeds the inner trip count via its
+; incremented value.
define void @addrec_non_unit_outer_stride(ptr %dst) {
; CHECK-LABEL: VPlan for loop in 'addrec_non_unit_outer_stride'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
@@ -403,9 +404,10 @@ define void @addrec_non_unit_outer_stride(ptr %dst) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<outer>:
; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
-; CHECK-NEXT: IR %0 = add i64 %outer.iv, 4
+; CHECK-NEXT: IR %0 = add i64 %outer.iv, 6
; CHECK-NEXT: IR %1 = udiv i64 %0, 3
; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: IR %outer.iv.next = add nuw i64 %outer.iv, 2
; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
@@ -415,6 +417,7 @@ entry:
outer:
%outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.iv.next = add nuw i64 %outer.iv, 2
br label %inner
inner:
@@ -422,14 +425,13 @@ inner:
%gep = getelementptr i8, ptr %dst, i64 %iv
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
- %bound = add i64 %outer.iv, 5
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %bound = add i64 %outer.iv.next, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
- %outer.iv.next = add nuw i64 %outer.iv, 2
- %cmp.outer = icmp ult i64 %outer.iv.next, 100
- br i1 %cmp.outer, label %outer, label %exit
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
@@ -464,8 +466,8 @@ inner:
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
%bound = add i64 %outer.iv, 5
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %mid.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %mid.latch
mid.latch:
%mid.next = add nuw i64 %mid, 1
@@ -508,8 +510,8 @@ inner:
%gep = getelementptr i8, ptr %dst, i64 %iv
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
- %cmp.inner = icmp ult i64 %iv.next, %scaled
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %scaled
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i64 %outer.iv, 1
@@ -519,7 +521,7 @@ exit:
ret void
}
-; {4,+,4}<nuw> from outer loop needs expansion.
+; {5,+,4}<nuw> from outer loop needs expansion.
define void @addrec_nuw_flags(ptr %dst) {
; CHECK-LABEL: VPlan for loop in 'addrec_nuw_flags'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
@@ -551,13 +553,105 @@ inner:
%gep = getelementptr i8, ptr %dst, i64 %iv
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; The outer loop has a non-affine recurrence %ar = {4,+,5,+,3}<outer> which
+; needs expanding as part of the trip count.
+define void @addrec_non_affine_outer_recurrence_no_canonical_iv(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_affine_outer_recurrence_no_canonical_iv'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %induction.iv = phi i64 [ %induction.iv.next, %outer.latch ], [ 9, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 5, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+; CHECK-NEXT: IR %umax = call i64 @llvm.umax.i64(i64 %induction.iv, i64 3)
+; CHECK-NEXT: IR %0 = add i64 %umax, -1
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 5, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %ar, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 3
+ %ar.next = add i64 %ar, %outer.iv
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; The outer loop has a canonical IV (%outer.iv) and non-affine recurrence
+; %ar = {4,+,5,+,3}<outer> which needs expanding as part of the trip count.
+define void @addrec_non_affine_outer_recurrence_with_canonical_iv(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_affine_outer_recurrence_with_canonical_iv'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %induction.iv = phi i64 [ %induction.iv.next, %outer.latch ], [ 9, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+; CHECK-NEXT: IR %umax = call i64 @llvm.umax.i64(i64 %induction.iv, i64 3)
+; CHECK-NEXT: IR %0 = add i64 %umax, -1
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %ar, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i64 %outer.iv, 1
- %cmp.outer = icmp ult i64 %outer.iv.next, 100
- br i1 %cmp.outer, label %outer, label %exit
+ %ar.next = add i64 %ar, %outer.iv
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
diff --git a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
index 0a8e4fccb715f..e18200d733e3d 100644
--- a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
@@ -404,7 +404,7 @@ exit:
ret void
}
-; SCEV expander creates a new canonical IV in the outer loop.
+; Outer loop does not have a canonical induction phi. SCEV expansion introduces one.
define void @test_expand_new_canonical_iv_non_zero_start(ptr %dst) {
; CHECK-LABEL: define void @test_expand_new_canonical_iv_non_zero_start(
; CHECK-SAME: ptr [[DST:%.*]]) {
@@ -412,11 +412,11 @@ define void @test_expand_new_canonical_iv_non_zero_start(ptr %dst) {
; CHECK-NEXT: br label %[[OUTER:.*]]
; CHECK: [[OUTER]]:
; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[O:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[O_NEXT:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDVAR]], 17
; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT: [[BOUND:%.*]] = add i64 [[OUTER_IV]], 8
+; CHECK-NEXT: [[BOUND:%.*]] = add i64 [[O]], 8
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
@@ -450,13 +450,13 @@ define void @test_expand_new_canonical_iv_non_zero_start(ptr %dst) {
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 3
-; CHECK-NEXT: [[CMP_INNER:%.*]] = icmp ult i64 [[IV_NEXT]], [[BOUND]]
-; CHECK-NEXT: br i1 [[CMP_INNER]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: [[EC_INNER:%.*]] = icmp ult i64 [[IV_NEXT]], [[BOUND]]
+; CHECK-NEXT: br i1 [[EC_INNER]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1
-; CHECK-NEXT: [[CMP_OUTER:%.*]] = icmp ult i64 [[OUTER_IV_NEXT]], 110
+; CHECK-NEXT: [[O_NEXT]] = add nuw i64 [[O]], 1
+; CHECK-NEXT: [[EC_OUTER:%.*]] = icmp ult i64 [[O_NEXT]], 110
; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT: br i1 [[CMP_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -473,13 +473,13 @@ inner:
%gep = getelementptr i8, ptr %dst, i64 %iv
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i64 %outer.iv, 1
- %cmp.outer = icmp ult i64 %outer.iv.next, 110
- br i1 %cmp.outer, label %outer, label %exit
+ %ec.outer = icmp ult i64 %outer.iv.next, 110
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
More information about the llvm-commits
mailing list