[llvm] [LV] Add early-exit-with-store tests (PR #140899)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Thu May 22 07:54:10 PDT 2025
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/140899
>From e00483923ad17e368377c74dcf652205f7489add Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 21 May 2025 12:49:42 +0000
Subject: [PATCH 1/4] [LV] Add early-exit-with-store tests
Adds some additional LoopVectorizeLegality tests for early exit loops
with a store that we don't vectorize.
---
.../LoopVectorize/early_exit_legality.ll | 186 ++++++++++++++++++
1 file changed, 186 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index de455c81d363e..b6a497fe4a672 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -470,6 +470,192 @@ loop.end:
ret i64 %retval
}
+define void @loop_contains_store_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_single_user'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_multi_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_multi_user'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %unused = add i16 %ee.val, 42
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_fcmp(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
+ %ee.val = load half, ptr %ee.addr, align 2
+ %ee.cond = fcmp ugt half %ee.val, 500.0
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %forward = getelementptr i16, ptr %pred, i64 -8
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %some.addr = getelementptr inbounds nuw i16, ptr %forward, i64 %iv
+ store i16 42, ptr %some.addr, align 2
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %n_bytes = mul nuw nsw i32 %n, 2
+ call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
+ %tc = sext i32 %n to i64
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, %tc
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_volatile'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store volatile i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @exit_conditions_combined(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'exit_conditions_combined'
+; CHECK: LV: Not vectorizing: Cannot vectorize uncountable loop.
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ %or.cond = select i1 %ee.cond, i1 true, i1 %counted.cond
+ br i1 %or.cond, label %exit, label %for.body
+
+exit: ; preds = %for.body
+ ret void
+}
define i64 @uncountable_exit_in_conditional_block(ptr %mask) {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_in_conditional_block'
>From 3f401f88e5aad3684d795357acbbaa3ed6361835 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 22 May 2025 10:50:35 +0000
Subject: [PATCH 2/4] * Move tests directly related to store functionality to a
new file * Improved test function names to clarify * Removed redundant test
---
.../LoopVectorize/early_exit_legality.ll | 195 +-----------------
.../early_exit_store_legality.ll | 168 +++++++++++++++
2 files changed, 170 insertions(+), 193 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index b6a497fe4a672..22b34079755c5 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -442,199 +442,8 @@ loop.end:
ret i64 %retval
}
-
-define i64 @loop_contains_store(ptr %dest) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops
-entry:
- %p1 = alloca [1024 x i8]
- call void @init_mem(ptr %p1, i64 1024)
- br label %loop
-
-loop:
- %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
- %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index
- %ld1 = load i32, ptr %arrayidx, align 1
- %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
- store i32 %ld1, ptr %arrayidx2, align 4
- %cmp = icmp eq i32 %ld1, 1
- br i1 %cmp, label %loop.inc, label %loop.end
-
-loop.inc:
- %index.next = add i64 %index, 1
- %exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
-
-loop.end:
- %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
- ret i64 %retval
-}
-
-define void @loop_contains_store_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_single_user'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
- %ee.val = load i16, ptr %ee.addr, align 2
- %ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @loop_contains_store_multi_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_multi_user'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
- %ee.val = load i16, ptr %ee.addr, align 2
- %ee.cond = icmp sgt i16 %ee.val, 500
- %unused = add i16 %ee.val, 42
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @loop_contains_store_fcmp(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
- %ee.val = load half, ptr %ee.addr, align 2
- %ee.cond = fcmp ugt half %ee.val, 500.0
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- %forward = getelementptr i16, ptr %pred, i64 -8
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
- %ee.val = load i16, ptr %ee.addr, align 2
- %ee.cond = icmp sgt i16 %ee.val, 500
- %some.addr = getelementptr inbounds nuw i16, ptr %forward, i64 %iv
- store i16 42, ptr %some.addr, align 2
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- %n_bytes = mul nuw nsw i32 %n, 2
- call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
- %tc = sext i32 %n to i64
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
- %ee.val = load i16, ptr %ee.addr, align 2
- %ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, %tc
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_volatile'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
- store volatile i16 %inc, ptr %st.addr, align 2
- %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
- %ee.val = load i16, ptr %ee.addr, align 2
- %ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
- %iv.next = add nuw nsw i64 %iv, 1
- %counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
-
-exit:
- ret void
-}
-
-define void @exit_conditions_combined(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'exit_conditions_combined'
+define void @exit_conditions_combined_in_single_branch(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'exit_conditions_combined_in_single_branch'
; CHECK: LV: Not vectorizing: Cannot vectorize uncountable loop.
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
new file mode 100644
index 0000000000000..f409bfc042e3f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+
+define i64 @loop_contains_store(ptr %dest) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops
+entry:
+ %p1 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index
+ %ld1 = load i32, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
+ store i32 %ld1, ptr %arrayidx2, align 4
+ %cmp = icmp eq i32 %ld1, 1
+ br i1 %cmp, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
+ %ee.val = load half, ptr %ee.addr, align 2
+ %ee.cond = fcmp ugt half %ee.val, 500.0
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %forward = getelementptr i16, ptr %pred, i64 -8
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %some.addr = getelementptr inbounds nuw i16, ptr %forward, i64 %iv
+ store i16 42, ptr %some.addr, align 2
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %n_bytes = mul nuw nsw i32 %n, 2
+ call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
+ %tc = sext i32 %n to i64
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, %tc
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_volatile'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store volatile i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+declare void @init_mem(ptr, i64);
>From e33de7d41a18f569fb98497068c84daeec92878e Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 22 May 2025 14:35:22 +0000
Subject: [PATCH 3/4] Added suggested tests
---
.../early_exit_store_legality.ll | 187 ++++++++++++++++++
1 file changed, 187 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index f409bfc042e3f..fb7ae5df8949b 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -55,6 +55,30 @@ exit:
ret void
}
+define void @loop_contains_store_ee_condition_is_invariant(ptr dereferenceable(40) noalias %array, i16 %ee.val) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_ee_condition_is_invariant'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
@@ -110,6 +134,36 @@ exit:
ret void
}
+define void @loop_contains_store_unsafe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unsafe_dependency'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %unknown.offset = call i64 @get_an_unknown_offset()
+ %unknown.base = getelementptr i16, ptr %pred, i64 %unknown.offset
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %some.addr = getelementptr inbounds nuw i16, ptr %unknown.base, i64 %iv
+ store i16 42, ptr %some.addr, align 2
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
@@ -139,6 +193,60 @@ exit:
ret void
}
+define void @loop_contains_store_to_pointer_with_no_deref_info(ptr noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_unknown_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ %n_bytes = mul nuw nsw i32 %n, 2
+ %tc = sext i32 %n to i64
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, %tc
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_volatile'
; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
@@ -165,4 +273,83 @@ exit:
ret void
}
+define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %store_addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ store i16 %inc, ptr %st.addr, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_requiring_alias_check(ptr dereferenceable(40) %array, ptr align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_requiring_alias_check'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
declare void @init_mem(ptr, i64);
+declare i64 @get_an_unknown_offset();
>From 936c9bbc7caf6f1daae0a174119303734d704e0e Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 22 May 2025 14:50:22 +0000
Subject: [PATCH 4/4] Add test for chained condition load with gather
---
.../early_exit_store_legality.ll | 28 +++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index fb7ae5df8949b..2edab3fa989e2 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -351,5 +351,33 @@ exit:
ret void
}
+define void @loop_contains_store_condition_load_is_chained(ptr dereferenceable(40) noalias %array, ptr align 8 dereferenceable(160) readonly %offsets, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_is_chained'
+; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %gather.addr = getelementptr inbounds nuw i64, ptr %offsets, i64 %iv
+ %ee.offset = load i64, ptr %gather.addr, align 8
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %ee.offset
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
declare void @init_mem(ptr, i64);
declare i64 @get_an_unknown_offset();
More information about the llvm-commits
mailing list