[llvm] [LV] Pre-committing tests for changing loop interleaving count computation (PR #70272)
Nilanjana Basu via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 10 16:16:15 PST 2023
https://github.com/nilanjana87 updated https://github.com/llvm/llvm-project/pull/70272
>From 8ee31fc228fdd07c0221004f10bf06c43e0beab9 Mon Sep 17 00:00:00 2001
From: Nilanjana Basu <n_basu at apple.com>
Date: Tue, 24 Oct 2023 15:48:46 -0700
Subject: [PATCH 1/3] [LV] Pre-committing tests for changing interleaving count
computation
---
.../LoopVectorize/AArch64/interleave_count.ll | 106 ++++++++++
.../LoopVectorize/X86/unroll-small-loops.ll | 194 +++++++++++++++++-
2 files changed, 296 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
new file mode 100644
index 000000000000000..74fd2f44fce0109
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -force-vector-width=64 -O3 -S -pass-remarks=loop-vectorize 2>&1 | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+%pair = type { i8, i8 }
+
+; For a loop with known trip count of 128, when we force VF 64, it should use
+; IC 2, since there is no remainder loop needed when the vector loop runs.
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
+define void @loop_with_tc_128(ptr %p, ptr %q) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr %pair, ptr %p, i64 %i, i32 0
+ %tmp1 = load i8, ptr %tmp0, align 1
+ %tmp2 = getelementptr %pair, ptr %p, i64 %i, i32 1
+ %tmp3 = load i8, ptr %tmp2, align 1
+ %add = add i8 %tmp1, %tmp3
+ %qi = getelementptr i8, ptr %q, i64 %i
+ store i8 %add, ptr %qi, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp eq i64 %i.next, 128
+ br i1 %cond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+; For a loop with known trip count of 129, when we force VF 64, it should use
+; IC 1, since there may be a remainder loop that needs to run after the vector loop.
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
+define void @loop_with_tc_129(ptr %p, ptr %q) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr %pair, ptr %p, i64 %i, i32 0
+ %tmp1 = load i8, ptr %tmp0, align 1
+ %tmp2 = getelementptr %pair, ptr %p, i64 %i, i32 1
+ %tmp3 = load i8, ptr %tmp2, align 1
+ %add = add i8 %tmp1, %tmp3
+ %qi = getelementptr i8, ptr %q, i64 %i
+ store i8 %add, ptr %qi, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp eq i64 %i.next, 129
+ br i1 %cond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+; For a loop with unknown trip count but a profile showing an approx TC estimate of 128,
+; when we force VF 64, it should use IC 2, since chances are high that the remainder loop
+; won't need to run
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
+define void @loop_with_profile_tc_128(ptr %p, ptr %q, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr %pair, ptr %p, i64 %i, i32 0
+ %tmp1 = load i8, ptr %tmp0, align 1
+ %tmp2 = getelementptr %pair, ptr %p, i64 %i, i32 1
+ %tmp3 = load i8, ptr %tmp2, align 1
+ %add = add i8 %tmp1, %tmp3
+ %qi = getelementptr i8, ptr %q, i64 %i
+ store i8 %add, ptr %qi, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %for.end, label %for.body, !prof !0
+
+for.end:
+ ret void
+}
+
+; For a loop with unknown trip count but a profile showing an approx TC estimate of 129,
+; when we force VF 64, it should use IC 1, since chances are high that the remainder loop
+; will need to run
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
+define void @loop_with_profile_tc_129(ptr %p, ptr %q, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr %pair, ptr %p, i64 %i, i32 0
+ %tmp1 = load i8, ptr %tmp0, align 1
+ %tmp2 = getelementptr %pair, ptr %p, i64 %i, i32 1
+ %tmp3 = load i8, ptr %tmp2, align 1
+ %add = add i8 %tmp1, %tmp3
+ %qi = getelementptr i8, ptr %q, i64 %i
+ store i8 %add, ptr %qi, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %for.end, label %for.body, !prof !1
+
+for.end:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 127}
+!1 = !{!"branch_weights", i32 1, i32 128}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
index 290be569bc12524..6b71cfef06be149 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
@@ -8,20 +8,20 @@ target triple = "x86_64-apple-macosx10.8.0"
; We don't unroll this loop because it has a small constant trip count.
;
-; CHECK-VECTOR-LABEL: @foo(
+; CHECK-VECTOR-LABEL: @foo_trip_count_8(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
; CHECK-VECTOR: store <4 x i32>
; CHECK-VECTOR-NOT: store <4 x i32>
; CHECK-VECTOR: ret
;
-; CHECK-SCALAR-LABEL: @foo(
+; CHECK-SCALAR-LABEL: @foo_trip_count_8(
; CHECK-SCALAR: load i32, ptr
; CHECK-SCALAR-NOT: load i32, ptr
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo(ptr nocapture %A) nounwind uwtable ssp {
+define i32 @foo_trip_count_8(ptr nocapture %A) nounwind uwtable ssp {
br label %1
; <label>:1 ; preds = %1, %0
@@ -32,7 +32,193 @@ define i32 @foo(ptr nocapture %A) nounwind uwtable ssp {
store i32 %4, ptr %2, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
- %exitcond = icmp eq i32 %lftr.wideiv, 100
+ %exitcond = icmp eq i32 %lftr.wideiv, 8
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_16(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_16(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_16(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 16
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_17(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_17(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_17(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 17
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_24(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_24(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_24(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 24
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_25(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_25(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_25(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 25
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_33(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_33(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_33(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 33
+ br i1 %exitcond, label %5, label %1
+
+; <label>:5 ; preds = %1
+ ret i32 undef
+}
+
+; CHECK-VECTOR-LABEL: @foo_trip_count_101(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo_trip_count_101(
+; CHECK-SCALAR: load i32, ptr
+; CHECK-SCALAR-NOT: load i32, ptr
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo_trip_count_101(ptr nocapture %A) nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %1, %0
+ %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+ %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %3 = load i32, ptr %2, align 4
+ %4 = add nsw i32 %3, 6
+ store i32 %4, ptr %2, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, 101
br i1 %exitcond, label %5, label %1
; <label>:5 ; preds = %1
>From 55d3431b8f19b8e55b177540d22ea24544c542cc Mon Sep 17 00:00:00 2001
From: Nilanjana Basu <n_basu at apple.com>
Date: Fri, 27 Oct 2023 19:05:32 -0700
Subject: [PATCH 2/3] Addressed reviewer comments
- replaced -O3 in tests with -p loop-vectorize
- added "TODO" in comments for tests that are meant to change after the subsequent patch
- used named labels for basic blocks
---
.../LoopVectorize/AArch64/interleave_count.ll | 6 +-
.../LoopVectorize/X86/unroll-small-loops.ll | 194 ++++++++++--------
2 files changed, 117 insertions(+), 83 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
index 74fd2f44fce0109..8b798ea3126b0a5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -force-vector-width=64 -O3 -S -pass-remarks=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt < %s -force-vector-width=64 -p loop-vectorize -S -pass-remarks=loop-vectorize 2>&1 | FileCheck %s
target triple = "aarch64-linux-gnu"
@@ -28,7 +28,7 @@ for.end:
ret void
}
-; For a loop with known trip count of 129, when we force VF 64, it should use
+; TODO: For a loop with known trip count of 129, when we force VF 64, it should use
; IC 1, since there may be a remainder loop that needs to run after the vector loop.
; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
define void @loop_with_tc_129(ptr %p, ptr %q) {
@@ -77,7 +77,7 @@ for.end:
ret void
}
-; For a loop with unknown trip count but a profile showing an approx TC estimate of 129,
+; TODO: For a loop with unknown trip count but a profile showing an approx TC estimate of 129,
; when we force VF 64, it should use IC 1, since chances are high that the remainder loop
; will need to run
; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
index 6b71cfef06be149..ab5fc7bc2e4782b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
@@ -6,7 +6,8 @@
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"
-; We don't unroll this loop because it has a small constant trip count.
+; We don't unroll this loop because it has a small constant trip count
+; that is not profitable for generating a scalar epilogue
;
; CHECK-VECTOR-LABEL: @foo_trip_count_8(
; CHECK-VECTOR: load <4 x i32>
@@ -21,24 +22,29 @@ target triple = "x86_64-apple-macosx10.8.0"
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_8(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_8(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 8
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop 4 times since TC being a multiple of VF means
+; that the epilogue loop may not need to run, making it profitable for
+; the vector loop to run even once
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_16(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -52,24 +58,29 @@ define i32 @foo_trip_count_8(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_16(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_16(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 16
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop twice since TC not being a multiple of VF may require
+; the epilogue loop to run, making it profitable when the vector loop runs
+; at least twice.
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_17(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -83,24 +94,30 @@ define i32 @foo_trip_count_16(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_17(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_17(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 17
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop 4 times since TC being a multiple of VF means
+; that the epilogue loop may not need to run, making it profitable for
+; the vector loop to run even once. The IC is restricted to 4 since
+; that is the maximum supported for the target.
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_24(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -114,24 +131,29 @@ define i32 @foo_trip_count_17(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_24(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_24(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 24
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop twice since TC not being a multiple of VF may require
+; the epilogue loop to run, making it profitable when the vector loop runs
+; at least twice.
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_25(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -145,24 +167,29 @@ define i32 @foo_trip_count_24(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_25(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_25(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 25
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop 4 times since TC not being a multiple of VF may require
+; the epilogue loop to run, making it profitable when the vector loop runs
+; at least twice.
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_33(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -176,24 +203,30 @@ define i32 @foo_trip_count_25(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_33(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_33(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 33
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
+; TODO: We unroll this loop 4 times since TC not being a multiple of VF may require
+; the epilogue loop to run, making it profitable when the vector loop runs
+; at least twice. The IC is restricted to 4 since that is the maximum supported
+; for the target.
+;
; CHECK-VECTOR-LABEL: @foo_trip_count_101(
; CHECK-VECTOR: load <4 x i32>
; CHECK-VECTOR-NOT: load <4 x i32>
@@ -207,22 +240,23 @@ define i32 @foo_trip_count_33(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @foo_trip_count_101(ptr nocapture %A) nounwind uwtable ssp {
- br label %1
+define void @foo_trip_count_101(ptr nocapture %A) nounwind uwtable ssp {
+entry:
+ br label %for.body
-; <label>:1 ; preds = %1, %0
- %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
- %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
- %3 = load i32, ptr %2, align 4
- %4 = add nsw i32 %3, 6
- store i32 %4, ptr %2, align 4
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %1 = load i32, ptr %0, align 4
+ %2 = add nsw i32 %1, 6
+ store i32 %2, ptr %0, align 4
%indvars.iv.next = add i64 %indvars.iv, 1
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
%exitcond = icmp eq i32 %lftr.wideiv, 101
- br i1 %exitcond, label %5, label %1
+ br i1 %exitcond, label %for.end, label %for.body
-; <label>:5 ; preds = %1
- ret i32 undef
+for.end: ; preds = %for.body
+ ret void
}
; But this is a good small loop to unroll as we don't know of a bound on its
@@ -239,7 +273,7 @@ define i32 @foo_trip_count_101(ptr nocapture %A) nounwind uwtable ssp {
; CHECK-SCALAR: store i32
; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
-define i32 @bar(ptr nocapture %A, i32 %n) nounwind uwtable ssp {
+define void @bar(ptr nocapture %A, i32 %n) nounwind uwtable ssp {
%1 = icmp sgt i32 %n, 0
br i1 %1, label %.lr.ph, label %._crit_edge
@@ -255,7 +289,7 @@ define i32 @bar(ptr nocapture %A, i32 %n) nounwind uwtable ssp {
br i1 %exitcond, label %._crit_edge, label %.lr.ph
._crit_edge: ; preds = %.lr.ph, %0
- ret i32 undef
+ ret void
}
; Also unroll if we need a runtime check but it was going to be added for
>From 205335aa6175e5b850e864ac48bb61f737b550ed Mon Sep 17 00:00:00 2001
From: Nilanjana Basu <n_basu at apple.com>
Date: Fri, 10 Nov 2023 16:12:21 -0800
Subject: [PATCH 3/3] Changed test VFs to a more reasonable value, fixed
comments
---
.../LoopVectorize/AArch64/interleave_count.ll | 43 ++++++++++---------
.../LoopVectorize/X86/unroll-small-loops.ll | 12 +++---
2 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
index 8b798ea3126b0a5..be747fb4c36d535 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave_count.ll
@@ -1,13 +1,14 @@
-; RUN: opt < %s -force-vector-width=64 -p loop-vectorize -S -pass-remarks=loop-vectorize 2>&1 | FileCheck %s
+; RUN: opt < %s -force-vector-width=8 -tiny-trip-count-interleave-threshold=16 -p loop-vectorize -S -pass-remarks=loop-vectorize 2>&1 | FileCheck %s
+; TODO: remove -tiny-trip-count-interleave-threshold once the interleave threshold is removed
target triple = "aarch64-linux-gnu"
%pair = type { i8, i8 }
-; For a loop with known trip count of 128, when we force VF 64, it should use
-; IC 2, since there is no remainder loop needed when the vector loop runs.
-; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
-define void @loop_with_tc_128(ptr %p, ptr %q) {
+; For a loop with known trip count of 16, when we force VF 8, it should use
+; IC 2 since there is no remainder loop run needed when the vector loop runs.
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 8, interleaved count: 2)
+define void @loop_with_tc_16(ptr noalias %p, ptr noalias %q) {
entry:
br label %for.body
@@ -21,17 +22,17 @@ for.body:
%qi = getelementptr i8, ptr %q, i64 %i
store i8 %add, ptr %qi, align 1
%i.next = add nuw nsw i64 %i, 1
- %cond = icmp eq i64 %i.next, 128
+ %cond = icmp eq i64 %i.next, 16
br i1 %cond, label %for.end, label %for.body
for.end:
ret void
}
-; TODO: For a loop with known trip count of 129, when we force VF 64, it should use
-; IC 1, since there may be a remainder loop that needs to run after the vector loop.
-; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
-define void @loop_with_tc_129(ptr %p, ptr %q) {
+; TODO: For a loop with known trip count of 17, when we force VF 8, it should use
+; IC 1 since there may be a remainder loop that needs to run after the vector loop.
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 8, interleaved count: 2)
+define void @loop_with_tc_17(ptr noalias %p, ptr noalias %q) {
entry:
br label %for.body
@@ -45,18 +46,18 @@ for.body:
%qi = getelementptr i8, ptr %q, i64 %i
store i8 %add, ptr %qi, align 1
%i.next = add nuw nsw i64 %i, 1
- %cond = icmp eq i64 %i.next, 129
+ %cond = icmp eq i64 %i.next, 17
br i1 %cond, label %for.end, label %for.body
for.end:
ret void
}
-; For a loop with unknown trip count but a profile showing an approx TC estimate of 128,
-; when we force VF 64, it should use IC 2, since chances are high that the remainder loop
+; For a loop with unknown trip count but a profile showing an approx TC estimate of 16,
+; when we force VF 8, it should use IC 2 since chances are high that the remainder loop
; won't need to run
-; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
-define void @loop_with_profile_tc_128(ptr %p, ptr %q, i64 %n) {
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 8, interleaved count: 2)
+define void @loop_with_profile_tc_16(ptr noalias %p, ptr noalias %q, i64 %n) {
entry:
br label %for.body
@@ -77,11 +78,11 @@ for.end:
ret void
}
-; TODO: For a loop with unknown trip count but a profile showing an approx TC estimate of 129,
-; when we force VF 64, it should use IC 1, since chances are high that the remainder loop
+; TODO: For a loop with unknown trip count but a profile showing an approx TC estimate of 17,
+; when we force VF 8, it should use IC 1 since chances are high that the remainder loop
; will need to run
-; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 64, interleaved count: 2)
-define void @loop_with_profile_tc_129(ptr %p, ptr %q, i64 %n) {
+; CHECK: remark: <unknown>:0:0: vectorized loop (vectorization width: 8, interleaved count: 2)
+define void @loop_with_profile_tc_17(ptr noalias %p, ptr noalias %q, i64 %n) {
entry:
br label %for.body
@@ -102,5 +103,5 @@ for.end:
ret void
}
-!0 = !{!"branch_weights", i32 1, i32 127}
-!1 = !{!"branch_weights", i32 1, i32 128}
+!0 = !{!"branch_weights", i32 1, i32 15}
+!1 = !{!"branch_weights", i32 1, i32 16}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
index ab5fc7bc2e4782b..5b79d6af9ed9ca5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
@@ -41,7 +41,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop 4 times since TC being a multiple of VF means
+; TODO: We should unroll this loop 4 times since TC being a multiple of VF means
; that the epilogue loop may not need to run, making it profitable for
; the vector loop to run even once
;
@@ -77,7 +77,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop twice since TC not being a multiple of VF may require
+; TODO: We should unroll this loop twice since TC not being a multiple of VF may require
; the epilogue loop to run, making it profitable when the vector loop runs
; at least twice.
;
@@ -113,7 +113,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop 4 times since TC being a multiple of VF means
+; TODO: We should unroll this loop 4 times since TC being a multiple of VF means
; that the epilogue loop may not need to run, making it profitable for
; the vector loop to run even once. The IC is restricted to 4 since
; that is the maximum supported for the target.
@@ -150,7 +150,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop twice since TC not being a multiple of VF may require
+; TODO: We should unroll this loop twice since TC not being a multiple of VF may require
; the epilogue loop to run, making it profitable when the vector loop runs
; at least twice.
;
@@ -186,7 +186,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop 4 times since TC not being a multiple of VF may require
+; TODO: We should unroll this loop 4 times since TC not being a multiple of VF may require
; the epilogue loop to run, making it profitable when the vector loop runs
; at least twice.
;
@@ -222,7 +222,7 @@ for.end: ; preds = %for.body
ret void
}
-; TODO: We unroll this loop 4 times since TC not being a multiple of VF may require
+; TODO: We should unroll this loop 4 times since TC not being a multiple of VF may require
; the epilogue loop to run, making it profitable when the vector loop runs
; at least twice. The IC is restricted to 4 since that is the maximum supported
; for the target.
More information about the llvm-commits
mailing list