[llvm] [LV] Add tests for argmin/argmax with a narrow index and ptr IVs (NFC) (PR #222290)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 03:32:19 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/222290
Extend test coverage for argmin/argmax with truncated IVs and pointer inductions.
>From 2917fd0230a1d4132f57ad5c051093084b9c26c6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 1 Sep 2026 08:38:58 +0100
Subject: [PATCH] [LV] Add tests for argmin/argmax with a narrow index and ptr
IVs (NFC)
Extend test coverage for argmin/argmax with truncated IVs and pointer
inductions.
---
.../AArch64/select-first-index-narrow.ll | 49 ++
.../LoopVectorize/find-last-ptr-induction.ll | 345 ++++++++++++
.../LoopVectorize/iv-select-cmp-trunc.ll | 266 ++++++++++
.../select-first-index-narrow.ll | 494 ++++++++++++++++++
4 files changed, 1154 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/select-first-index-narrow.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/select-first-index-narrow.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-first-index-narrow.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-first-index-narrow.ll
new file mode 100644
index 0000000000000..3fe714ce9d9f6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-first-index-narrow.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+; Selects canonical IV truncated to i32.
+define i32 @argmin_s32_i32idx_natural_vf(ptr %a) {
+; CHECK-LABEL: define i32 @argmin_s32_i32idx_natural_vf(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[L]], [[MIN_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i32 @llvm.smin.i32(i32 [[L]], i32 [[MIN_VAL]])
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %min.val = phi i32 [ 0, %entry ], [ %min.val.next, %loop ]
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp slt i32 %l, %min.val
+ %t = trunc i64 %iv to i32
+ %min.idx.next = select i1 %cmp, i32 %t, i32 %min.idx
+ %iv.next = add i64 %iv, 1
+ %min.val.next = tail call i32 @llvm.smin.i32(i32 %l, i32 %min.val)
+ %ec = icmp eq i64 %iv.next, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %min.idx.next, %loop ]
+ ret i32 %r
+}
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll b/llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll
new file mode 100644
index 0000000000000..2cc0581e6b629
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll
@@ -0,0 +1,345 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+target datalayout = "e-p1:32:32"
+
+; Pointer induction stepping over the array that is loaded from.
+define ptr @find_last_ptr_induction(ptr %a, i64 %n) {
+; CHECK-LABEL: define ptr @find_last_ptr_induction(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x ptr> [ splat (ptr null), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i64 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x ptr> [[VECTOR_GEP]], <4 x ptr> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call ptr @llvm.experimental.vector.extract.last.active.v4p0(<4 x ptr> [[TMP9]], <4 x i1> [[TMP8]], ptr null)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi ptr [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ null, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LAST:%.*]] = phi ptr [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[LAST_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], 3
+; CHECK-NEXT: [[LAST_NEXT]] = select i1 [[CMP]], ptr [[P]], ptr [[LAST]]
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i32, ptr [[P]], i64 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi ptr [ [[LAST_NEXT]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret ptr [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = phi ptr [ %a, %entry ], [ %p.next, %loop ]
+ %last = phi ptr [ null, %entry ], [ %last.next, %loop ]
+ %l = load i32, ptr %p
+ %cmp = icmp sgt i32 %l, 3
+ %last.next = select i1 %cmp, ptr %p, ptr %last
+ %p.next = getelementptr inbounds i32, ptr %p, i64 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi ptr [ %last.next, %loop ]
+ ret ptr %r
+}
+
+; Pointer induction and loaded array are independent; non-null start value.
+define ptr @find_last_ptr_induction_separate_array(ptr %a, ptr %b, ptr %init, i64 %n) {
+; CHECK-LABEL: define ptr @find_last_ptr_induction_separate_array(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[INIT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N_VEC]], 24
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[INIT]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[B]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x ptr> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 24, i64 48, i64 72>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x ptr> [[VECTOR_GEP]], <4 x ptr> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 96
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x ptr> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = call ptr @llvm.experimental.vector.extract.last.active.v4p0(<4 x ptr> [[TMP9]], <4 x i1> [[TMP8]], ptr [[TMP11]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi ptr [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LAST:%.*]] = phi ptr [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[LAST_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], 3
+; CHECK-NEXT: [[LAST_NEXT]] = select i1 [[CMP]], ptr [[P]], ptr [[LAST]]
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 24
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi ptr [ [[LAST_NEXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret ptr [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = phi ptr [ %b, %entry ], [ %p.next, %loop ]
+ %last = phi ptr [ %init, %entry ], [ %last.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, 3
+ %last.next = select i1 %cmp, ptr %p, ptr %last
+ %p.next = getelementptr inbounds i8, ptr %p, i64 24
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi ptr [ %last.next, %loop ]
+ ret ptr %r
+}
+
+; Selected pointer is not a pointer induction.
+define ptr @find_last_ptr_load(ptr %a, i64 %n) {
+; CHECK-LABEL: define ptr @find_last_ptr_load(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x ptr> [ splat (ptr null), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x ptr>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x ptr> [[WIDE_LOAD]], splat (ptr null)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x ptr> [[WIDE_LOAD]], <4 x ptr> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call ptr @llvm.experimental.vector.extract.last.active.v4p0(<4 x ptr> [[TMP7]], <4 x i1> [[TMP6]], ptr null)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi ptr [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ null, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LAST:%.*]] = phi ptr [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[LAST_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne ptr [[L]], null
+; CHECK-NEXT: [[LAST_NEXT]] = select i1 [[CMP]], ptr [[L]], ptr [[LAST]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi ptr [ [[LAST_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret ptr [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %last = phi ptr [ null, %entry ], [ %last.next, %loop ]
+ %gep = getelementptr inbounds ptr, ptr %a, i64 %iv
+ %l = load ptr, ptr %gep
+ %cmp = icmp ne ptr %l, null
+ %last.next = select i1 %cmp, ptr %l, ptr %last
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi ptr [ %last.next, %loop ]
+ ret ptr %r
+}
+
+; Pointer index paired with a min/max reduction (argmax).
+define ptr @argmax_ptr_index(ptr %a, i32 %init, i64 %n) {
+; CHECK-LABEL: define ptr @argmax_ptr_index(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_PTR:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MAX_PTR_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[MAX_PTR_NEXT]] = select i1 [[CMP]], ptr [[P]], ptr [[MAX_PTR]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i32, ptr [[P]], i64 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi ptr [ [[MAX_PTR_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret ptr [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %p = phi ptr [ %a, %entry ], [ %p.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.ptr = phi ptr [ null, %entry ], [ %max.ptr.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %max.ptr.next = select i1 %cmp, ptr %p, ptr %max.ptr
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %p.next = getelementptr inbounds i32, ptr %p, i64 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi ptr [ %max.ptr.next, %loop ]
+ ret ptr %r
+}
+
+; Pointer induction step is not the canonical IV type.
+define ptr addrspace(1) @find_last_ptr_induction_narrow_step(ptr addrspace(1) %a, i64 %n) {
+; CHECK-LABEL: define ptr addrspace(1) @find_last_ptr_induction_narrow_step(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i32 [[TMP2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr addrspace(1) [ [[A]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x ptr addrspace(1)> [ splat (ptr addrspace(1) null), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr addrspace(1) [[POINTER_PHI]], <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr addrspace(1)> [[VECTOR_GEP]], i64 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP8]], <4 x i1> [[TMP6]], <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP10]] = select i1 [[TMP8]], <4 x ptr addrspace(1)> [[VECTOR_GEP]], <4 x ptr addrspace(1)> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr addrspace(1) [[POINTER_PHI]], i32 16
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call ptr addrspace(1) @llvm.experimental.vector.extract.last.active.v4p1(<4 x ptr addrspace(1)> [[TMP10]], <4 x i1> [[TMP9]], ptr addrspace(1) null)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr addrspace(1) [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi ptr addrspace(1) [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ null, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr addrspace(1) [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LAST:%.*]] = phi ptr addrspace(1) [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[LAST_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr addrspace(1) [[PTR_IV]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 3
+; CHECK-NEXT: [[LAST_NEXT]] = select i1 [[C]], ptr addrspace(1) [[PTR_IV]], ptr addrspace(1) [[LAST]]
+; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR_IV]], i32 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi ptr addrspace(1) [ [[LAST_NEXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret ptr addrspace(1) [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr.iv = phi ptr addrspace(1) [ %a, %entry ], [ %ptr.iv.next, %loop ]
+ %last = phi ptr addrspace(1) [ null, %entry ], [ %last.next, %loop ]
+ %l = load i32, ptr addrspace(1) %ptr.iv, align 4
+ %c = icmp eq i32 %l, 3
+ %last.next = select i1 %c, ptr addrspace(1) %ptr.iv, ptr addrspace(1) %last
+ %ptr.iv.next = getelementptr inbounds i32, ptr addrspace(1) %ptr.iv, i32 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi ptr addrspace(1) [ %last.next, %loop ]
+ ret ptr addrspace(1) %r
+}
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index 8e132d62a436f..58ffa4c050973 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -2409,3 +2409,269 @@ loop:
exit:
ret i1 %sel
}
+
+; The i64 induction is also widened for the store.
+define i32 @select_icmp_truncated_iv_also_widened(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-VF4IC1-LABEL: define i32 @select_icmp_truncated_iv_also_widened(
+; CHECK-VF4IC1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC1-NEXT: [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF4IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC1: [[VECTOR_PH]]:
+; CHECK-VF4IC1-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-VF4IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC1: [[VECTOR_BODY]]:
+; CHECK-VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT: [[VEC_IND1:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4IC1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-VF4IC1-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC1-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-VF4IC1-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-VF4IC1-NEXT: [[TMP6]] = select i1 [[TMP9]], <4 x i1> [[TMP2]], <4 x i1> [[TMP3]]
+; CHECK-VF4IC1-NEXT: [[TMP7]] = select i1 [[TMP9]], <4 x i32> [[VEC_IND1]], <4 x i32> [[VEC_PHI]]
+; CHECK-VF4IC1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4IC1-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP4]], align 8
+; CHECK-VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF4IC1-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC1-NEXT: [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
+; CHECK-VF4IC1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-VF4IC1: [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC1-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP7]], <4 x i1> [[TMP6]], i32 331)
+; CHECK-VF4IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC1: [[SCALAR_PH]]:
+; CHECK-VF4IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; CHECK-VF4IC1-NEXT: br label %[[LOOP:.*]]
+; CHECK-VF4IC1: [[LOOP]]:
+; CHECK-VF4IC1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC1-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-VF4IC1-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], 3
+; CHECK-VF4IC1-NEXT: [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF4IC1-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[RDX]]
+; CHECK-VF4IC1-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-VF4IC1-NEXT: store i64 [[IV]], ptr [[GEP_B]], align 8
+; CHECK-VF4IC1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC1-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC1-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-VF4IC1: [[EXIT]]:
+; CHECK-VF4IC1-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC1-NEXT: ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF4IC4-LABEL: define i32 @select_icmp_truncated_iv_also_widened(
+; CHECK-VF4IC4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF4IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-VF4IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF4IC4: [[VECTOR_PH]]:
+; CHECK-VF4IC4-NEXT: [[TMP0:%.*]] = and i64 [[N]], 15
+; CHECK-VF4IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF4IC4: [[VECTOR_BODY]]:
+; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 331), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP11:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[VEC_IND4:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD5:%.*]] = add <4 x i32> [[VEC_IND4]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_26:%.*]] = add <4 x i32> [[STEP_ADD5]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[STEP_ADD_37:%.*]] = add <4 x i32> [[STEP_ADD_26]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 12
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-VF4IC4-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-VF4IC4-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP6:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD4]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP7:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD5]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 3)
+; CHECK-VF4IC4-NEXT: [[TMP29:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-VF4IC4-NEXT: [[TMP30:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-VF4IC4-NEXT: [[TMP31:%.*]] = or <4 x i1> [[TMP29]], [[TMP30]]
+; CHECK-VF4IC4-NEXT: [[TMP32:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-VF4IC4-NEXT: [[TMP33:%.*]] = or <4 x i1> [[TMP31]], [[TMP32]]
+; CHECK-VF4IC4-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP8]]
+; CHECK-VF4IC4-NEXT: [[TMP19:%.*]] = or <4 x i1> [[TMP33]], [[TMP18]]
+; CHECK-VF4IC4-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]])
+; CHECK-VF4IC4-NEXT: [[TMP21]] = select i1 [[TMP20]], <4 x i1> [[TMP5]], <4 x i1> [[TMP9]]
+; CHECK-VF4IC4-NEXT: [[TMP22]] = select i1 [[TMP20]], <4 x i1> [[TMP6]], <4 x i1> [[TMP10]]
+; CHECK-VF4IC4-NEXT: [[TMP23]] = select i1 [[TMP20]], <4 x i1> [[TMP7]], <4 x i1> [[TMP11]]
+; CHECK-VF4IC4-NEXT: [[TMP24]] = select i1 [[TMP20]], <4 x i1> [[TMP8]], <4 x i1> [[TMP12]]
+; CHECK-VF4IC4-NEXT: [[TMP25]] = select i1 [[TMP20]], <4 x i32> [[VEC_IND4]], <4 x i32> [[VEC_PHI]]
+; CHECK-VF4IC4-NEXT: [[TMP26]] = select i1 [[TMP20]], <4 x i32> [[STEP_ADD5]], <4 x i32> [[VEC_PHI1]]
+; CHECK-VF4IC4-NEXT: [[TMP27]] = select i1 [[TMP20]], <4 x i32> [[STEP_ADD_26]], <4 x i32> [[VEC_PHI2]]
+; CHECK-VF4IC4-NEXT: [[TMP28]] = select i1 [[TMP20]], <4 x i32> [[STEP_ADD_37]], <4 x i32> [[VEC_PHI3]]
+; CHECK-VF4IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF4IC4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 4
+; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 8
+; CHECK-VF4IC4-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 12
+; CHECK-VF4IC4-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP13]], align 8
+; CHECK-VF4IC4-NEXT: store <4 x i64> [[STEP_ADD]], ptr [[TMP14]], align 8
+; CHECK-VF4IC4-NEXT: store <4 x i64> [[STEP_ADD_2]], ptr [[TMP15]], align 8
+; CHECK-VF4IC4-NEXT: store <4 x i64> [[STEP_ADD_3]], ptr [[TMP16]], align 8
+; CHECK-VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD_3]], splat (i64 4)
+; CHECK-VF4IC4-NEXT: [[VEC_IND_NEXT11]] = add <4 x i32> [[STEP_ADD_37]], splat (i32 4)
+; CHECK-VF4IC4-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-VF4IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF4IC4-NEXT: [[TMP34:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP25]], <4 x i1> [[TMP21]], i32 331)
+; CHECK-VF4IC4-NEXT: [[TMP35:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP26]], <4 x i1> [[TMP22]], i32 [[TMP34]])
+; CHECK-VF4IC4-NEXT: [[TMP36:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP27]], <4 x i1> [[TMP23]], i32 [[TMP35]])
+; CHECK-VF4IC4-NEXT: [[TMP37:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP28]], <4 x i1> [[TMP24]], i32 [[TMP36]])
+; CHECK-VF4IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF4IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF4IC4: [[SCALAR_PH]]:
+; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP37]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; CHECK-VF4IC4-NEXT: br label %[[LOOP:.*]]
+; CHECK-VF4IC4: [[LOOP]]:
+; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF4IC4-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-VF4IC4-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], 3
+; CHECK-VF4IC4-NEXT: [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF4IC4-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[RDX]]
+; CHECK-VF4IC4-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-VF4IC4-NEXT: store i64 [[IV]], ptr [[GEP_B]], align 8
+; CHECK-VF4IC4-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF4IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF4IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-VF4IC4: [[EXIT]]:
+; CHECK-VF4IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP37]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF4IC4-NEXT: ret i32 [[SEL_LCSSA]]
+;
+; CHECK-VF1IC4-LABEL: define i32 @select_icmp_truncated_iv_also_widened(
+; CHECK-VF1IC4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-VF1IC4-NEXT: [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-VF1IC4-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-VF1IC4: [[VECTOR_PH]]:
+; CHECK-VF1IC4-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-VF1IC4-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-VF1IC4-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-VF1IC4: [[VECTOR_BODY]]:
+; CHECK-VF1IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP37:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP38:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 331, %[[VECTOR_PH]] ], [ [[TMP39:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP17:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP18:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP19:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
+; CHECK-VF1IC4-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 3
+; CHECK-VF1IC4-NEXT: [[TMP20:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-VF1IC4-NEXT: [[TMP21:%.*]] = add i32 [[TMP20]], 0
+; CHECK-VF1IC4-NEXT: [[TMP41:%.*]] = add i32 [[TMP20]], 1
+; CHECK-VF1IC4-NEXT: [[TMP42:%.*]] = add i32 [[TMP20]], 2
+; CHECK-VF1IC4-NEXT: [[TMP43:%.*]] = add i32 [[TMP20]], 3
+; CHECK-VF1IC4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-VF1IC4-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT: [[TMP14:%.*]] = icmp sgt i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT: [[TMP15:%.*]] = icmp sgt i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT: [[TMP16:%.*]] = icmp sgt i32 [[TMP12]], 3
+; CHECK-VF1IC4-NEXT: [[TMP44:%.*]] = freeze i1 [[TMP13]]
+; CHECK-VF1IC4-NEXT: [[TMP26:%.*]] = freeze i1 [[TMP14]]
+; CHECK-VF1IC4-NEXT: [[TMP27:%.*]] = or i1 [[TMP44]], [[TMP26]]
+; CHECK-VF1IC4-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP15]]
+; CHECK-VF1IC4-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
+; CHECK-VF1IC4-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP16]]
+; CHECK-VF1IC4-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
+; CHECK-VF1IC4-NEXT: [[TMP32]] = select i1 [[TMP31]], i1 [[TMP13]], i1 [[TMP1]]
+; CHECK-VF1IC4-NEXT: [[TMP33]] = select i1 [[TMP31]], i1 [[TMP14]], i1 [[TMP17]]
+; CHECK-VF1IC4-NEXT: [[TMP34]] = select i1 [[TMP31]], i1 [[TMP15]], i1 [[TMP18]]
+; CHECK-VF1IC4-NEXT: [[TMP35]] = select i1 [[TMP31]], i1 [[TMP16]], i1 [[TMP19]]
+; CHECK-VF1IC4-NEXT: [[TMP36]] = select i1 [[TMP31]], i32 [[TMP21]], i32 [[VEC_PHI]]
+; CHECK-VF1IC4-NEXT: [[TMP37]] = select i1 [[TMP31]], i32 [[TMP41]], i32 [[VEC_PHI1]]
+; CHECK-VF1IC4-NEXT: [[TMP38]] = select i1 [[TMP31]], i32 [[TMP42]], i32 [[VEC_PHI2]]
+; CHECK-VF1IC4-NEXT: [[TMP39]] = select i1 [[TMP31]], i32 [[TMP43]], i32 [[VEC_PHI3]]
+; CHECK-VF1IC4-NEXT: [[TMP40:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-VF1IC4-NEXT: [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-VF1IC4-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
+; CHECK-VF1IC4-NEXT: [[TMP24:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP4]]
+; CHECK-VF1IC4-NEXT: store i64 [[INDEX]], ptr [[TMP40]], align 8
+; CHECK-VF1IC4-NEXT: store i64 [[TMP2]], ptr [[TMP22]], align 8
+; CHECK-VF1IC4-NEXT: store i64 [[TMP3]], ptr [[TMP23]], align 8
+; CHECK-VF1IC4-NEXT: store i64 [[TMP4]], ptr [[TMP24]], align 8
+; CHECK-VF1IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-VF1IC4-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-VF1IC4: [[MIDDLE_BLOCK]]:
+; CHECK-VF1IC4-NEXT: [[TMP45:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 331
+; CHECK-VF1IC4-NEXT: [[TMP46:%.*]] = select i1 [[TMP33]], i32 [[TMP37]], i32 [[TMP45]]
+; CHECK-VF1IC4-NEXT: [[TMP47:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP46]]
+; CHECK-VF1IC4-NEXT: [[TMP48:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP47]]
+; CHECK-VF1IC4-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-VF1IC4-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-VF1IC4: [[SCALAR_PH]]:
+; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; CHECK-VF1IC4-NEXT: br label %[[LOOP:.*]]
+; CHECK-VF1IC4: [[LOOP]]:
+; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-VF1IC4-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-VF1IC4-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], 3
+; CHECK-VF1IC4-NEXT: [[TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-VF1IC4-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[RDX]]
+; CHECK-VF1IC4-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-VF1IC4-NEXT: store i64 [[IV]], ptr [[GEP_B]], align 8
+; CHECK-VF1IC4-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-VF1IC4-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-VF1IC4-NEXT: br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-VF1IC4: [[EXIT]]:
+; CHECK-VF1IC4-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP48]], %[[MIDDLE_BLOCK]] ]
+; CHECK-VF1IC4-NEXT: ret i32 [[SEL_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 331, %entry ], [ %sel, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep.a, align 4
+ %cmp = icmp sgt i32 %l, 3
+ %trunc = trunc i64 %iv to i32
+ %sel = select i1 %cmp, i32 %trunc, i32 %rdx
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %iv, ptr %gep.b, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit.cond = icmp eq i64 %iv.next, %n
+ br i1 %exit.cond, label %exit, label %loop
+
+exit:
+ ret i32 %sel
+}
diff --git a/llvm/test/Transforms/LoopVectorize/select-first-index-narrow.ll b/llvm/test/Transforms/LoopVectorize/select-first-index-narrow.ll
new file mode 100644
index 0000000000000..5ce87f6b0fcd0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/select-first-index-narrow.ll
@@ -0,0 +1,494 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; i32 element, i32 index, strict argmax (signed).
+define i32 @argmax_s32_i32idx(ptr %a, i32 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmax_s32_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i32 element, i32 index, strict argmin (signed).
+define i32 @argmin_s32_i32idx(ptr %a, i32 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmin_s32_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[L]], [[MIN_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i32 @llvm.smin.i32(i32 [[L]], i32 [[MIN_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %min.val = phi i32 [ %init, %entry ], [ %min.val.next, %loop ]
+ %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp slt i32 %l, %min.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %min.idx.next = select i1 %cmp, i32 %t, i32 %min.idx
+ %min.val.next = tail call i32 @llvm.smin.i32(i32 %l, i32 %min.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %min.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i32 element, i32 index, strict argmax (unsigned).
+define i32 @argmax_u32_i32idx(ptr %a, i32 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmax_u32_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.umax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp ugt i32 %l, %max.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i32 @llvm.umax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i8 element, i32 index, strict argmax (signed).
+define i32 @argmax_s8_i32idx(ptr %a, i8 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmax_s8_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i8 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i8 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i8 @llvm.smax.i8(i8 [[L]], i8 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i8 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %a, i64 %iv
+ %l = load i8, ptr %gep
+ %cmp = icmp sgt i8 %l, %max.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i8 @llvm.smax.i8(i8 %l, i8 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i16 element, i32 index, strict argmax (signed).
+define i32 @argmax_s16_i32idx(ptr %a, i16 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmax_s16_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i16 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i16 @llvm.smax.i16(i16 [[L]], i16 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i16 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i16, ptr %a, i64 %iv
+ %l = load i16, ptr %gep
+ %cmp = icmp sgt i16 %l, %max.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i16 @llvm.smax.i16(i16 %l, i16 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i64 element, i32 index, strict argmax (signed).
+define i32 @argmax_s64_i32idx(ptr %a, i64 %init, i32 %n) {
+; CHECK-LABEL: define i32 @argmax_s64_i32idx(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C:%.*]] = icmp sgt i32 [[N]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT: [[N_EXT:%.*]] = zext nneg i32 [[N]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[L]], i64 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ %c = icmp sgt i32 %n, 1
+ call void @llvm.assume(i1 %c)
+ %n.ext = zext nneg i32 %n to i64
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i64 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %l = load i64, ptr %gep
+ %cmp = icmp sgt i64 %l, %max.val
+ %t = trunc nuw nsw i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i64 @llvm.smax.i64(i64 %l, i64 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n.ext
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; i32 index with an unconstrained i64 trip count.
+define i32 @argmax_s32_i32idx_unknown_i64_tc(ptr %a, i32 %init, i64 %n) {
+; CHECK-LABEL: define i32 @argmax_s32_i32idx_unknown_i64_tc(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %t = trunc i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
+
+; Sentinel boundary: i16 index at trip count 65536 hits the UMin sentinel.
+; FIXME: currently indistinguishable from below_sentinel: both bail out in
+; handleFirstArgMinOrMax before the sentinel is ever computed.
+define i32 @argmax_s32_i16idx_at_sentinel(ptr %a, i32 %init) {
+; CHECK-LABEL: define i32 @argmax_s32_i16idx_at_sentinel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i16
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i16 [[T]], i16 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 65536
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i16 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RE:%.*]] = zext i16 [[R]] to i32
+; CHECK-NEXT: ret i32 [[RE]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i16 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %t = trunc i64 %iv to i16
+ %max.idx.next = select i1 %cmp, i16 %t, i16 %max.idx
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 65536
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i16 [ %max.idx.next, %loop ]
+ %re = zext i16 %r to i32
+ ret i32 %re
+}
+
+; Sentinel boundary: i16 index at trip count 65535, just below the sentinel.
+define i32 @argmax_s32_i16idx_below_sentinel(ptr %a, i32 %init) {
+; CHECK-LABEL: define i32 @argmax_s32_i16idx_below_sentinel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i16
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i16 [[T]], i16 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 65535
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i16 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RE:%.*]] = zext i16 [[R]] to i32
+; CHECK-NEXT: ret i32 [[RE]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i16 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %t = trunc i64 %iv to i16
+ %max.idx.next = select i1 %cmp, i16 %t, i16 %max.idx
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 65535
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i16 [ %max.idx.next, %loop ]
+ %re = zext i16 %r to i32
+ ret i32 %re
+}
+
+; A non-canonical IV (start 5, step 3) with a narrow index.
+define i32 @argmax_s32_i32idx_non_canonical_iv(ptr %a, i32 %init) {
+; CHECK-LABEL: define i32 @argmax_s32_i32idx_non_canonical_iv(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[INIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 5, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i32 [[T]], i32 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i32 @llvm.smax.i32(i32 [[L]], i32 [[MAX_VAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 3
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 3005
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 5, %entry ], [ %iv.next, %loop ]
+ %max.val = phi i32 [ %init, %entry ], [ %max.val.next, %loop ]
+ %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep
+ %cmp = icmp sgt i32 %l, %max.val
+ %t = trunc i64 %iv to i32
+ %max.idx.next = select i1 %cmp, i32 %t, i32 %max.idx
+ %max.val.next = tail call i32 @llvm.smax.i32(i32 %l, i32 %max.val)
+ %iv.next = add nuw nsw i64 %iv, 3
+ %ec = icmp eq i64 %iv.next, 3005
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %r = phi i32 [ %max.idx.next, %loop ]
+ ret i32 %r
+}
More information about the llvm-commits
mailing list