[llvm] b176c11 - [LV] Add tests for missed scalarization opportunities. (#225508)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 00:57:49 PDT 2026
Author: Florian Hahn
Date: 2026-09-23T08:57:42+01:00
New Revision: b176c11a5e77d32be272a528b4fc8ee24840a8d5
URL: https://github.com/llvm/llvm-project/commit/b176c11a5e77d32be272a528b4fc8ee24840a8d5
DIFF: https://github.com/llvm/llvm-project/commit/b176c11a5e77d32be272a528b4fc8ee24840a8d5.diff
LOG: [LV] Add tests for missed scalarization opportunities. (#225508)
Add additional test coverage with unnecessary extracts.
Added:
llvm/test/Transforms/LoopVectorize/early-exit-live-out-extract-lane.ll
Modified:
llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-live-out-extract-lane.ll b/llvm/test/Transforms/LoopVectorize/early-exit-live-out-extract-lane.ll
new file mode 100644
index 0000000000000..80b1cda71aa12
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-live-out-extract-lane.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck --check-prefix=VF1IC2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck --check-prefix=VF4IC2 %s
+
+declare void @init_mem(ptr, i64)
+
+; The condition selecting the early exit is loop-invariant, so after unrolling
+; all sources of the extract-lane are the same single scalar.
+define i64 @early_exit_with_live_in_condition(i1 %cond) {
+; VF1IC2-LABEL: define i64 @early_exit_with_live_in_condition(
+; VF1IC2-SAME: i1 [[COND:%.*]]) {
+; VF1IC2-NEXT: [[ENTRY:.*:]]
+; VF1IC2-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
+; VF1IC2-NEXT: call void @init_mem(ptr [[P]], i64 1024)
+; VF1IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF1IC2: [[VECTOR_PH]]:
+; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF1IC2: [[VECTOR_BODY]]:
+; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; VF1IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
+; VF1IC2-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
+; VF1IC2-NEXT: [[TMP5:%.*]] = icmp eq i8 [[TMP3]], 0
+; VF1IC2-NEXT: [[TMP6:%.*]] = icmp eq i8 [[TMP4]], 0
+; VF1IC2-NEXT: [[TMP7:%.*]] = select i1 [[COND]], i1 true, i1 [[TMP5]]
+; VF1IC2-NEXT: [[TMP8:%.*]] = select i1 [[COND]], i1 true, i1 [[TMP6]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP7]]
+; VF1IC2-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP8]]
+; VF1IC2-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF1IC2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF1IC2-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF1IC2: [[VECTOR_BODY_INTERIM]]:
+; VF1IC2-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF1IC2: [[MIDDLE_BLOCK]]:
+; VF1IC2-NEXT: br label %[[LOOP_END:.*]]
+; VF1IC2: [[VECTOR_EARLY_EXIT_CHECK]]:
+; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i1 [[TMP8]], false
+; VF1IC2-NEXT: [[TMP14:%.*]] = zext i1 [[TMP13]] to i64
+; VF1IC2-NEXT: [[TMP15:%.*]] = add i64 1, [[TMP14]]
+; VF1IC2-NEXT: [[TMP16:%.*]] = icmp eq i1 [[TMP7]], false
+; VF1IC2-NEXT: [[TMP17:%.*]] = zext i1 [[TMP16]] to i64
+; VF1IC2-NEXT: [[TMP18:%.*]] = add i64 0, [[TMP17]]
+; VF1IC2-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP17]], 1
+; VF1IC2-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 [[TMP15]]
+; VF1IC2-NEXT: [[TMP21:%.*]] = sub i64 [[TMP20]], 1
+; VF1IC2-NEXT: [[TMP22:%.*]] = icmp uge i64 [[TMP20]], 1
+; VF1IC2-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i1 [[COND]], i1 [[COND]]
+; VF1IC2-NEXT: br i1 [[TMP23]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; VF1IC2: [[VECTOR_EARLY_EXIT_1]]:
+; VF1IC2-NEXT: br label %[[LOOP_END]]
+; VF1IC2: [[VECTOR_EARLY_EXIT_0]]:
+; VF1IC2-NEXT: br label %[[LOOP_END]]
+; VF1IC2: [[LOOP_END]]:
+; VF1IC2-NEXT: [[RETVAL:%.*]] = phi i64 [ 1, %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[VECTOR_EARLY_EXIT_0]] ], [ 2, %[[MIDDLE_BLOCK]] ]
+; VF1IC2-NEXT: ret i64 [[RETVAL]]
+;
+; VF4IC2-LABEL: define i64 @early_exit_with_live_in_condition(
+; VF4IC2-SAME: i1 [[COND:%.*]]) {
+; VF4IC2-NEXT: [[ENTRY:.*:]]
+; VF4IC2-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
+; VF4IC2-NEXT: call void @init_mem(ptr [[P]], i64 1024)
+; VF4IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4IC2: [[VECTOR_PH]]:
+; VF4IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
+; VF4IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; VF4IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4IC2: [[VECTOR_BODY]]:
+; VF4IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; VF4IC2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; VF4IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
+; VF4IC2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; VF4IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; VF4IC2-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; VF4IC2-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], zeroinitializer
+; VF4IC2-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP2]]
+; VF4IC2-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; VF4IC2-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP4]]
+; VF4IC2-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP5]]
+; VF4IC2-NEXT: [[TMP8:%.*]] = or <4 x i1> [[TMP6]], [[TMP7]]
+; VF4IC2-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; VF4IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; VF4IC2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF4IC2-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF4IC2: [[VECTOR_BODY_INTERIM]]:
+; VF4IC2-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4IC2: [[MIDDLE_BLOCK]]:
+; VF4IC2-NEXT: br label %[[LOOP_END:.*]]
+; VF4IC2: [[VECTOR_EARLY_EXIT_CHECK]]:
+; VF4IC2-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; VF4IC2-NEXT: [[TMP11:%.*]] = add i64 4, [[FIRST_ACTIVE_LANE]]
+; VF4IC2-NEXT: [[FIRST_ACTIVE_LANE2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; VF4IC2-NEXT: [[TMP12:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE2]]
+; VF4IC2-NEXT: [[TMP13:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE2]], 4
+; VF4IC2-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[TMP11]]
+; VF4IC2-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i64 [[TMP14]]
+; VF4IC2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 4
+; VF4IC2-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i64 [[TMP16]]
+; VF4IC2-NEXT: [[TMP18:%.*]] = icmp uge i64 [[TMP14]], 4
+; VF4IC2-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i1 [[TMP17]], i1 [[TMP15]]
+; VF4IC2-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; VF4IC2: [[VECTOR_EARLY_EXIT_1]]:
+; VF4IC2-NEXT: br label %[[LOOP_END]]
+; VF4IC2: [[VECTOR_EARLY_EXIT_0]]:
+; VF4IC2-NEXT: br label %[[LOOP_END]]
+; VF4IC2: [[LOOP_END]]:
+; VF4IC2-NEXT: [[RETVAL:%.*]] = phi i64 [ 1, %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[VECTOR_EARLY_EXIT_0]] ], [ 2, %[[MIDDLE_BLOCK]] ]
+; VF4IC2-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p = alloca [1024 x i8]
+ call void @init_mem(ptr %p, i64 1024)
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ]
+ br i1 %cond, label %loop.end, label %search
+
+search:
+ %arrayidx = getelementptr inbounds i8, ptr %p, i64 %iv
+ %ld = load i8, ptr %arrayidx, align 1
+ %cmp = icmp eq i8 %ld, 0
+ br i1 %cmp, label %loop.end, label %loop.latch
+
+loop.latch:
+ %inc = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, 1024
+ br i1 %exitcond, label %loop.end, label %loop.header
+
+loop.end:
+ %retval = phi i64 [ 0, %loop.header ], [ 1, %search ], [ 2, %loop.latch ]
+ ret i64 %retval
+}
+
+; With a scalar VF, the exit value is a
diff erent single scalar in each part,
+; so the extract-lane must be retained.
+define i8 @early_exit_live_out_with_distinct_scalar_parts() {
+; VF1IC2-LABEL: define i8 @early_exit_live_out_with_distinct_scalar_parts() {
+; VF1IC2-NEXT: [[ENTRY:.*:]]
+; VF1IC2-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
+; VF1IC2-NEXT: call void @init_mem(ptr [[P]], i64 1024)
+; VF1IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF1IC2: [[VECTOR_PH]]:
+; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF1IC2: [[VECTOR_BODY]]:
+; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; VF1IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
+; VF1IC2-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
+; VF1IC2-NEXT: [[TMP5:%.*]] = icmp sgt i8 [[TMP3]], 10
+; VF1IC2-NEXT: [[TMP6:%.*]] = icmp sgt i8 [[TMP4]], 10
+; VF1IC2-NEXT: [[TMP7:%.*]] = freeze i1 [[TMP5]]
+; VF1IC2-NEXT: [[TMP8:%.*]] = freeze i1 [[TMP6]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF1IC2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF1IC2-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF1IC2: [[VECTOR_BODY_INTERIM]]:
+; VF1IC2-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF1IC2: [[MIDDLE_BLOCK]]:
+; VF1IC2-NEXT: br label %[[EXIT:.*]]
+; VF1IC2: [[VECTOR_EARLY_EXIT]]:
+; VF1IC2-NEXT: [[TMP11:%.*]] = mul i8 [[TMP3]], 7
+; VF1IC2-NEXT: [[TMP12:%.*]] = mul i8 [[TMP4]], 7
+; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i1 [[TMP6]], false
+; VF1IC2-NEXT: [[TMP14:%.*]] = zext i1 [[TMP13]] to i64
+; VF1IC2-NEXT: [[TMP15:%.*]] = add i64 1, [[TMP14]]
+; VF1IC2-NEXT: [[TMP16:%.*]] = icmp eq i1 [[TMP5]], false
+; VF1IC2-NEXT: [[TMP17:%.*]] = zext i1 [[TMP16]] to i64
+; VF1IC2-NEXT: [[TMP18:%.*]] = add i64 0, [[TMP17]]
+; VF1IC2-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP17]], 1
+; VF1IC2-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 [[TMP15]]
+; VF1IC2-NEXT: [[TMP21:%.*]] = sub i64 [[TMP20]], 1
+; VF1IC2-NEXT: [[TMP22:%.*]] = icmp uge i64 [[TMP20]], 1
+; VF1IC2-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i8 [[TMP12]], i8 [[TMP11]]
+; VF1IC2-NEXT: br label %[[EARLY:.*]]
+; VF1IC2: [[EARLY]]:
+; VF1IC2-NEXT: ret i8 [[TMP23]]
+; VF1IC2: [[EXIT]]:
+; VF1IC2-NEXT: ret i8 0
+;
+; VF4IC2-LABEL: define i8 @early_exit_live_out_with_distinct_scalar_parts() {
+; VF4IC2-NEXT: [[ENTRY:.*:]]
+; VF4IC2-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
+; VF4IC2-NEXT: call void @init_mem(ptr [[P]], i64 1024)
+; VF4IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4IC2: [[VECTOR_PH]]:
+; VF4IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4IC2: [[VECTOR_BODY]]:
+; VF4IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; VF4IC2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; VF4IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
+; VF4IC2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; VF4IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; VF4IC2-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i8> [[WIDE_LOAD]], splat (i8 10)
+; VF4IC2-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i8> [[WIDE_LOAD2]], splat (i8 10)
+; VF4IC2-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
+; VF4IC2-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP3]]
+; VF4IC2-NEXT: [[TMP6:%.*]] = or <4 x i1> [[TMP4]], [[TMP5]]
+; VF4IC2-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; VF4IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; VF4IC2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF4IC2-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; VF4IC2: [[VECTOR_BODY_INTERIM]]:
+; VF4IC2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF4IC2: [[MIDDLE_BLOCK]]:
+; VF4IC2-NEXT: br label %[[EXIT:.*]]
+; VF4IC2: [[VECTOR_EARLY_EXIT]]:
+; VF4IC2-NEXT: [[TMP9:%.*]] = mul <4 x i8> [[WIDE_LOAD]], splat (i8 7)
+; VF4IC2-NEXT: [[TMP10:%.*]] = mul <4 x i8> [[WIDE_LOAD2]], splat (i8 7)
+; VF4IC2-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; VF4IC2-NEXT: [[TMP11:%.*]] = add i64 4, [[FIRST_ACTIVE_LANE]]
+; VF4IC2-NEXT: [[FIRST_ACTIVE_LANE3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; VF4IC2-NEXT: [[TMP12:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE3]]
+; VF4IC2-NEXT: [[TMP13:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE3]], 4
+; VF4IC2-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[TMP11]]
+; VF4IC2-NEXT: [[TMP15:%.*]] = extractelement <4 x i8> [[TMP9]], i64 [[TMP14]]
+; VF4IC2-NEXT: [[TMP16:%.*]] = sub i64 [[TMP14]], 4
+; VF4IC2-NEXT: [[TMP17:%.*]] = extractelement <4 x i8> [[TMP10]], i64 [[TMP16]]
+; VF4IC2-NEXT: [[TMP18:%.*]] = icmp uge i64 [[TMP14]], 4
+; VF4IC2-NEXT: [[TMP19:%.*]] = select i1 [[TMP18]], i8 [[TMP17]], i8 [[TMP15]]
+; VF4IC2-NEXT: br label %[[EARLY:.*]]
+; VF4IC2: [[EARLY]]:
+; VF4IC2-NEXT: ret i8 [[TMP19]]
+; VF4IC2: [[EXIT]]:
+; VF4IC2-NEXT: ret i8 0
+;
+entry:
+ %p = alloca [1024 x i8]
+ call void @init_mem(ptr %p, i64 1024)
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p, i64 %iv
+ %ld = load i8, ptr %arrayidx, align 1
+ %scaled = mul i8 %ld, 7
+ %cmp = icmp sgt i8 %ld, 10
+ br i1 %cmp, label %early, label %loop.latch
+
+loop.latch:
+ %inc = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, 1024
+ br i1 %exitcond, label %exit, label %loop.header
+
+early:
+ ret i8 %scaled
+
+exit:
+ ret i8 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
index adf02a9a7f1fd..1685f3e6ace39 100644
--- a/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
+++ b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
@@ -393,3 +393,195 @@ loop:
exit:
ret void
}
+
+; The select is only used by the extract computing the exit value.
+define i32 @narrow_select_used_by_extract_lane(ptr noalias %src) {
+; VF4IC1-LABEL: define i32 @narrow_select_used_by_extract_lane(
+; VF4IC1-SAME: ptr noalias [[SRC:%.*]]) {
+; VF4IC1-NEXT: [[ENTRY:.*:]]
+; VF4IC1-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; VF4IC1-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; VF4IC1-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], <4 x i32> zeroinitializer, <4 x i32> splat (i32 2)
+; VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4IC1: [[VECTOR_BODY]]:
+; VF4IC1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; VF4IC1-NEXT: br label %[[EXIT:.*]]
+; VF4IC1: [[EXIT]]:
+; VF4IC1-NEXT: ret i32 [[TMP3]]
+;
+; VF2IC2-LABEL: define i32 @narrow_select_used_by_extract_lane(
+; VF2IC2-SAME: ptr noalias [[SRC:%.*]]) {
+; VF2IC2-NEXT: [[ENTRY:.*:]]
+; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; VF2IC2-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; VF2IC2-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], <2 x i32> zeroinitializer, <2 x i32> splat (i32 2)
+; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2IC2: [[VECTOR_BODY]]:
+; VF2IC2-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> <i1 false, i1 true>, i1 false)
+; VF2IC2-NEXT: [[TMP11:%.*]] = add i64 2, [[FIRST_INACTIVE_LANE]]
+; VF2IC2-NEXT: [[FIRST_INACTIVE_LANE1:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> zeroinitializer, i1 false)
+; VF2IC2-NEXT: [[TMP4:%.*]] = add i64 0, [[FIRST_INACTIVE_LANE1]]
+; VF2IC2-NEXT: [[TMP5:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE1]], 2
+; VF2IC2-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 [[TMP11]]
+; VF2IC2-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP6]], 1
+; VF2IC2-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP2]], i64 [[LAST_ACTIVE_LANE]]
+; VF2IC2-NEXT: [[TMP8:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 2
+; VF2IC2-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP2]], i64 [[TMP8]]
+; VF2IC2-NEXT: [[TMP10:%.*]] = icmp uge i64 [[LAST_ACTIVE_LANE]], 2
+; VF2IC2-NEXT: [[TMP3:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; VF2IC2-NEXT: br label %[[EXIT:.*]]
+; VF2IC2: [[EXIT]]:
+; VF2IC2-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %l = load i32, ptr %src, align 4
+ %c = icmp eq i32 %l, 0
+ %sel = select i1 %c, i32 0, i32 2
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp ult i32 %iv, 2
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret i32 %sel
+}
+
+; The vector select uses the live-in condition directly, so narrowing it would
+; introduce a broadcast of the result without removing the broadcast of any of
+; its operands.
+define void @dont_narrow_select_with_live_in_condition(i1 %c, ptr noalias %dst) {
+; VF4IC1-LABEL: define void @dont_narrow_select_with_live_in_condition(
+; VF4IC1-SAME: i1 [[C:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4IC1-NEXT: [[ENTRY:.*:]]
+; VF4IC1-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1-NEXT: [[TMP0:%.*]] = select i1 [[C]], i32 0, i32 2
+; VF4IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
+; VF4IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4IC1: [[VECTOR_BODY]]:
+; VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; VF4IC1-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4
+; VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4IC1-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4IC1-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: br label %[[EXIT:.*]]
+; VF4IC1: [[EXIT]]:
+; VF4IC1-NEXT: ret void
+;
+; VF2IC2-LABEL: define void @dont_narrow_select_with_live_in_condition(
+; VF2IC2-SAME: i1 [[C:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2IC2-NEXT: [[ENTRY:.*:]]
+; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2-NEXT: [[TMP0:%.*]] = select i1 [[C]], i32 0, i32 2
+; VF2IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; VF2IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2IC2: [[VECTOR_BODY]]:
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; VF2IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
+; VF2IC2-NEXT: store <2 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4
+; VF2IC2-NEXT: store <2 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4
+; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF2IC2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2IC2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: br label %[[EXIT:.*]]
+; VF2IC2: [[EXIT]]:
+; VF2IC2-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sel = select i1 %c, i32 0, i32 2
+ %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %sel, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Same as above, but with the single-scalar condition defined in the loop.
+define void @dont_narrow_select_with_uniform_load_condition(ptr noalias %src, ptr noalias %dst) {
+; VF4IC1-LABEL: define void @dont_narrow_select_with_uniform_load_condition(
+; VF4IC1-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4IC1-NEXT: [[ENTRY:.*:]]
+; VF4IC1-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4IC1: [[VECTOR_BODY]]:
+; VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4IC1-NEXT: [[TMP0:%.*]] = load i1, ptr [[SRC]], align 1
+; VF4IC1-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i32 0, i32 2
+; VF4IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
+; VF4IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; VF4IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; VF4IC1-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4
+; VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4IC1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4IC1-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: br label %[[EXIT:.*]]
+; VF4IC1: [[EXIT]]:
+; VF4IC1-NEXT: ret void
+;
+; VF2IC2-LABEL: define void @dont_narrow_select_with_uniform_load_condition(
+; VF2IC2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2IC2-NEXT: [[ENTRY:.*:]]
+; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2IC2: [[VECTOR_BODY]]:
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2IC2-NEXT: [[TMP0:%.*]] = load i1, ptr [[SRC]], align 1
+; VF2IC2-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i32 0, i32 2
+; VF2IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0
+; VF2IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; VF2IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; VF2IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 2
+; VF2IC2-NEXT: store <2 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4
+; VF2IC2-NEXT: store <2 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF2IC2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2IC2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: br label %[[EXIT:.*]]
+; VF2IC2: [[EXIT]]:
+; VF2IC2-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %c = load i1, ptr %src, align 1
+ %sel = select i1 %c, i32 0, i32 2
+ %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %sel, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list