[llvm] 4d37b5b - [LV] Add additional SCEV expansion tests with AddRecs. (NFC) (#210159)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 04:21:52 PDT 2026
Author: Florian Hahn
Date: 2026-07-17T11:21:45Z
New Revision: 4d37b5bf3099d099302cce447168b3ece4efef5e
URL: https://github.com/llvm/llvm-project/commit/4d37b5bf3099d099302cce447168b3ece4efef5e
DIFF: https://github.com/llvm/llvm-project/commit/4d37b5bf3099d099302cce447168b3ece4efef5e.diff
LOG: [LV] Add additional SCEV expansion tests with AddRecs. (NFC) (#210159)
Add additional test coverage for SCEV expansion of AddRecs, including
cases where extends are needed, different start and steps, as well as
cases where the outer header is not the plan's entry block.
Added:
Modified:
llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
index 1e55658ce19ae..89279ea5629dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
@@ -345,13 +345,313 @@ inner:
store i8 0, ptr %gep
%iv.next = add nuw i64 %iv, 3
%bound = add i64 %outer.iv, 5
- %cmp.inner = icmp ult i64 %iv.next, %bound
- br i1 %cmp.inner, label %inner, label %outer.latch
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
outer.latch:
%outer.iv.next = add nuw i64 %outer.iv, 1
- %cmp.outer = icmp ult i64 %outer.iv.next, 100
- br i1 %cmp.outer, label %outer, label %exit
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+define void @addrec_outer_iv_narrow(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_outer_iv_narrow'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%0> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %indvar = phi i64 [ %indvar.next, %outer.latch ], [ 0, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = add i64 %indvar, 5
+; CHECK-NEXT: IR %ext = zext i32 %outer.iv to i64
+; CHECK-NEXT: Successor(s): vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ext = zext i32 %outer.iv to i64
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 1
+ %bound = add i64 %ext, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i32 %outer.iv, 1
+ %ec.outer = icmp ult i32 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; AddRec {0,+,2}<nuw> from the outer loop feeds the inner trip count via its
+; incremented value.
+define void @addrec_non_unit_outer_stride(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_unit_outer_stride'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = add i64 %outer.iv, 6
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: IR %outer.iv.next = add nuw i64 %outer.iv, 2
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %outer.iv.next = add nuw i64 %outer.iv, 2
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %outer.iv.next, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; AddRec in outer-most loop needs expansion.
+define void @addrec_over_grandparent_loop(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_over_grandparent_loop'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<middle>:
+; CHECK-NEXT: IR %mid = phi i64 [ 0, %outermost ], [ %mid.next, %mid.latch ]
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outermost
+
+outermost:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.iv.latch ]
+ br label %middle
+
+middle:
+ %mid = phi i64 [ 0, %outermost ], [ %mid.next, %mid.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %middle ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %outer.iv, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %mid.latch
+
+mid.latch:
+ %mid.next = add nuw i64 %mid, 1
+ %cm = icmp ult i64 %mid.next, 50
+ br i1 %cm, label %middle, label %outer.iv.latch
+
+outer.iv.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %co = icmp ult i64 %outer.iv.next, 100
+ br i1 %co, label %outermost, label %exit
+
+exit:
+ ret void
+}
+
+define void @addrec_phi_not_in_inner_preheader(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_phi_not_in_inner_preheader'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<inner.ph>:
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %scaled = add i64 %outer.iv, 5
+ %cc = icmp ult i64 %outer.iv, 1000
+ br i1 %cc, label %inner.ph, label %exit
+
+inner.ph:
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %inner.ph ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %ec.inner = icmp ult i64 %iv.next, %scaled
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ br label %outer.header
+
+exit:
+ ret void
+}
+
+; {5,+,4}<nuw> from outer loop needs expansion.
+define void @addrec_nuw_flags(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_nuw_flags'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%3> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %0 = shl nuw nsw i64 %outer.iv, 2
+; CHECK-NEXT: IR %1 = add i64 %0, 4
+; CHECK-NEXT: IR %2 = udiv i64 %1, 3
+; CHECK-NEXT: IR %3 = add nuw nsw i64 %2, 1
+; CHECK-NEXT: IR %m = mul nuw i64 %outer.iv, 4
+; CHECK-NEXT: IR %bound = add nuw i64 %m, 5
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%3>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %m = mul nuw i64 %outer.iv, 4
+ %bound = add nuw i64 %m, 5
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; The outer loop has a non-affine recurrence %ar = {4,+,5,+,3}<outer> which
+; needs expanding as part of the trip count.
+define void @addrec_non_affine_outer_recurrence_no_canonical_iv(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_affine_outer_recurrence_no_canonical_iv'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %induction.iv = phi i64 [ %induction.iv.next, %outer.latch ], [ 9, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 5, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+; CHECK-NEXT: IR %umax = call i64 @llvm.umax.i64(i64 %induction.iv, i64 3)
+; CHECK-NEXT: IR %0 = add i64 %umax, -1
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 5, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %ar, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 3
+ %ar.next = add i64 %ar, %outer.iv
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
+; The outer loop has a canonical IV (%outer.iv) and non-affine recurrence
+; %ar = {4,+,5,+,3}<outer> which needs expanding as part of the trip count.
+define void @addrec_non_affine_outer_recurrence_with_canonical_iv(ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'addrec_non_affine_outer_recurrence_with_canonical_iv'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%2> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<outer>:
+; CHECK-NEXT: IR %induction.iv = phi i64 [ %induction.iv.next, %outer.latch ], [ 9, %entry ]
+; CHECK-NEXT: IR %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT: IR %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+; CHECK-NEXT: IR %umax = call i64 @llvm.umax.i64(i64 %induction.iv, i64 3)
+; CHECK-NEXT: IR %0 = add i64 %umax, -1
+; CHECK-NEXT: IR %1 = udiv i64 %0, 3
+; CHECK-NEXT: IR %2 = add nuw nsw i64 %1, 1
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%2>, ir<4>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %ar = phi i64 [ 4, %entry ], [ %ar.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %bound = add i64 %ar, 5
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %ar.next = add i64 %ar, %outer.iv
+ %ec.outer = icmp ult i64 %outer.iv.next, 100
+ br i1 %ec.outer, label %outer, label %exit
exit:
ret void
diff --git a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
index 491882b796ef1..e18200d733e3d 100644
--- a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
@@ -404,4 +404,85 @@ exit:
ret void
}
+; Outer loop does not have a canonical induction phi. SCEV expansion introduces one.
+define void @test_expand_new_canonical_iv_non_zero_start(ptr %dst) {
+; CHECK-LABEL: define void @test_expand_new_canonical_iv_non_zero_start(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER:.*]]
+; CHECK: [[OUTER]]:
+; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[O:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[O_NEXT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDVAR]], 17
+; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[BOUND:%.*]] = add i64 [[O]], 8
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 3
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], 6
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 9
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT: store i8 0, ptr [[TMP8]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP9]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP10]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP3]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 3
+; CHECK-NEXT: [[EC_INNER:%.*]] = icmp ult i64 [[IV_NEXT]], [[BOUND]]
+; CHECK-NEXT: br i1 [[EC_INNER]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[O_NEXT]] = add nuw i64 [[O]], 1
+; CHECK-NEXT: [[EC_OUTER:%.*]] = icmp ult i64 [[O_NEXT]], 110
+; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT: br i1 [[EC_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer
+
+outer:
+ %outer.iv = phi i64 [ 10, %entry ], [ %outer.iv.next, %outer.latch ]
+ %bound = add i64 %outer.iv, 8
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep = getelementptr i8, ptr %dst, i64 %iv
+ store i8 0, ptr %gep
+ %iv.next = add nuw i64 %iv, 3
+ %ec.inner = icmp ult i64 %iv.next, %bound
+ br i1 %ec.inner, label %inner, label %outer.latch
+
+outer.latch:
+ %outer.iv.next = add nuw i64 %outer.iv, 1
+ %ec.outer = icmp ult i64 %outer.iv.next, 110
+ br i1 %ec.outer, label %outer, label %exit
+
+exit:
+ ret void
+}
+
declare void @cond()
More information about the llvm-commits
mailing list