[llvm] [LV] Keep predicate select when optimizeFindIVReductions when folding tail. (PR #191166)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 20:28:20 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/191166
>From 41f3bdf20105100eba999726562e28f7f5aa74a2 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 9 Apr 2026 04:02:49 -0700
Subject: [PATCH 1/4] Precommit tests.
---
.../LoopVectorize/iv-select-cmp-fold-tail.ll | 126 ++++++++++++++++++
1 file changed, 126 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index 619b0d28be229..260d2d0686a39 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -97,3 +97,129 @@ loop:
exit:
ret i32 %rdx.next
}
+
+define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
+; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_non_const_start(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 -1, i64 -2, i64 -3>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP50:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -1)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP12]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]
+; CHECK: [[PRED_LOAD_IF6]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP18]], i32 1
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP22]], i32 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_CONTINUE7]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP13]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP19]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i64> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
+; CHECK: [[PRED_LOAD_IF8]]:
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP27]]
+; CHECK-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP29]], i32 2
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP33:%.*]] = load i64, ptr [[TMP32]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP33]], i32 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]
+; CHECK: [[PRED_LOAD_CONTINUE9]]:
+; CHECK-NEXT: [[TMP35:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP30]], %[[PRED_LOAD_IF8]] ]
+; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x i64> [ [[TMP25]], %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP34]], %[[PRED_LOAD_IF8]] ]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; CHECK-NEXT: br i1 [[TMP37]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11]]
+; CHECK: [[PRED_LOAD_IF10]]:
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = load i64, ptr [[TMP39]], align 8
+; CHECK-NEXT: [[TMP41:%.*]] = insertelement <4 x i64> [[TMP35]], i64 [[TMP40]], i32 3
+; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP42]]
+; CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr [[TMP43]], align 8
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> [[TMP36]], i64 [[TMP44]], i32 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]
+; CHECK: [[PRED_LOAD_CONTINUE11]]:
+; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x i64> [ [[TMP35]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP41]], %[[PRED_LOAD_IF10]] ]
+; CHECK-NEXT: [[TMP47:%.*]] = phi <4 x i64> [ [[TMP36]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP45]], %[[PRED_LOAD_IF10]] ]
+; CHECK-NEXT: [[TMP48:%.*]] = icmp sgt <4 x i64> [[TMP46]], [[TMP47]]
+; CHECK-NEXT: [[TMP49]] = select <4 x i1> [[TMP48]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP50]] = or <4 x i1> [[VEC_PHI3]], [[TMP48]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
+; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP53:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP49]])
+; CHECK-NEXT: [[TMP54:%.*]] = add nsw i64 [[TMP53]], -1
+; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP50]])
+; CHECK-NEXT: [[TMP57:%.*]] = freeze i1 [[TMP56]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP57]], i64 [[TMP54]], i64 [[RDX_START]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[RDX_SELECT]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %entry, %loop
+ %iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ]
+ %rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ]
+ %iv.next = add nsw i64 %iv, -1
+ %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv.next
+ %ld.a = load i64, ptr %gep.a.iv, align 8
+ %gep.b.iv = getelementptr inbounds i64, ptr %b, i64 %iv.next
+ %ld.b = load i64, ptr %gep.b.iv, align 8
+ %cmp.a.b = icmp sgt i64 %ld.a, %ld.b
+ %cond = select i1 %cmp.a.b, i64 %iv.next, i64 %rdx
+ %exit.cond = icmp ugt i64 %iv, 1
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i64 %cond
+}
>From 64e70211abed1aef1d7e9f561c8d65fb7eefcd57 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 9 Apr 2026 03:05:12 -0700
Subject: [PATCH 2/4] [LV] Keep predicate select when optimizeFindIVReductions
when folding tail.
This patch recover the predicate select for FindIV reductions when
folding tail.
Note that the TC in
llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll is small
(12) which is lower than 16 (4 * 4), so the condition of predicated select are
constants.
Caused by https://github.com/llvm/llvm-project/pull/183911.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 +++++++++++-
.../LoopVectorize/iv-select-cmp-decreasing.ll | 20 +++++++++++++------
.../LoopVectorize/iv-select-cmp-fold-tail.ll | 6 ++++--
3 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 094eab7ca9246..3479559e792c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5799,6 +5799,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
// select.
VPValue *BackedgeVal = PhiR->getBackedgeValue();
auto *FindLastSelect = cast<VPSingleDefRecipe>(BackedgeVal);
+ auto *PredSelect =
+ HeaderMask ? cast<VPSingleDefRecipe>(BackedgeVal) : nullptr;
if (HeaderMask &&
!match(BackedgeVal,
m_Select(m_Specific(HeaderMask),
@@ -5900,6 +5902,10 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
}
}
+ // Update the predicated select with FindLastSelect when folding tail.
+ if (HeaderMask)
+ PredSelect->setOperand(1, FindLastSelect);
+
// Create the reduction result in the middle block using sentinel directly.
RecurKind MinMaxKind =
UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
@@ -5909,7 +5915,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
DebugLoc ExitDL = RdxResult->getDebugLoc();
VPBuilder MiddleBuilder(RdxResult);
VPValue *ReducedIV = MiddleBuilder.createNaryOp(
- VPInstruction::ComputeReductionResult, FindLastSelect, Flags, ExitDL);
+ VPInstruction::ComputeReductionResult,
+ HeaderMask ? PredSelect : FindLastSelect, Flags, ExitDL);
// If IVOfExpressionToSink is an expression to sink, sink it now.
VPValue *VectorRegionExitingVal = ReducedIV;
@@ -5949,6 +5956,10 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
/*IsInLoop=*/false, FastMathFlags());
auto *OrReduce = MiddleBuilder.createNaryOp(
VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
+ // Add predicated select to prevent poison value when folding tail.
+ if (HeaderMask)
+ OrVal = MiddleBuilder.createSelect(HeaderMask, OrVal, AnyOfPhi, ExitDL);
+
NewRdxResult = MiddleBuilder.createNaryOp(
VPInstruction::ComputeAnyOfResult,
{StartVPV, VectorRegionExitingVal, OrReduce}, {}, ExitDL);
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index b8ef3b0e464e9..c3ed8ae9ae4e2 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -332,9 +332,13 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
-; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
+; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
@@ -594,9 +598,13 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
-; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
+; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index 260d2d0686a39..6727e6b6fb2b4 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -195,9 +195,11 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP53:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP49]])
+; CHECK-NEXT: [[TMP52:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP49]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP53:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP52]])
; CHECK-NEXT: [[TMP54:%.*]] = add nsw i64 [[TMP53]], -1
-; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP50]])
+; CHECK-NEXT: [[TMP55:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP50]], <4 x i1> [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
; CHECK-NEXT: [[TMP57:%.*]] = freeze i1 [[TMP56]]
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP57]], i64 [[TMP54]], i64 [[RDX_START]]
; CHECK-NEXT: br label %[[EXIT:.*]]
>From ec328867dae1e9d6e16b23853e827c27adde9b31 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 9 Apr 2026 17:39:20 -0700
Subject: [PATCH 3/4] Fixup, using mask FindLast cond with header mask.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 ++++++-----
.../LoopVectorize/iv-select-cmp-decreasing.ll | 32 +++++++++----------
.../LoopVectorize/iv-select-cmp-fold-tail.ll | 10 +++---
3 files changed, 31 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3479559e792c7..3de00a5982001 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5799,8 +5799,6 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
// select.
VPValue *BackedgeVal = PhiR->getBackedgeValue();
auto *FindLastSelect = cast<VPSingleDefRecipe>(BackedgeVal);
- auto *PredSelect =
- HeaderMask ? cast<VPSingleDefRecipe>(BackedgeVal) : nullptr;
if (HeaderMask &&
!match(BackedgeVal,
m_Select(m_Specific(HeaderMask),
@@ -5880,6 +5878,15 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
}
+ // When tail folding, mask the condition with the header mask to prevent
+ // propagating poison from inactive lanes in the last vector iteration.
+ if (HeaderMask) {
+ VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
+ auto *Or = LoopBuilder.createLogicalAnd(HeaderMask,
+ FindLastSelect->getOperand(0));
+ FindLastSelect->setOperand(0, Or);
+ }
+
VPInstruction *RdxResult = cast<VPInstruction>(vputils::findRecipe(
BackedgeVal,
match_fn(m_VPInstruction<VPInstruction::ComputeReductionResult>())));
@@ -5902,10 +5909,6 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
}
}
- // Update the predicated select with FindLastSelect when folding tail.
- if (HeaderMask)
- PredSelect->setOperand(1, FindLastSelect);
-
// Create the reduction result in the middle block using sentinel directly.
RecurKind MinMaxKind =
UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
@@ -5915,8 +5918,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
DebugLoc ExitDL = RdxResult->getDebugLoc();
VPBuilder MiddleBuilder(RdxResult);
VPValue *ReducedIV = MiddleBuilder.createNaryOp(
- VPInstruction::ComputeReductionResult,
- HeaderMask ? PredSelect : FindLastSelect, Flags, ExitDL);
+ VPInstruction::ComputeReductionResult, FindLastSelect, Flags, ExitDL);
// If IVOfExpressionToSink is an expression to sink, sink it now.
VPValue *VectorRegionExitingVal = ReducedIV;
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index c3ed8ae9ae4e2..a53b5c0e560ea 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -326,16 +326,16 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = icmp ugt <4 x i16> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = icmp ugt <4 x i16> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = icmp ugt <4 x i16> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP100]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP101]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP102]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> zeroinitializer, <4 x i1> [[TMP103]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> [[TMP76]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> [[TMP77]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> [[TMP70]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> [[TMP71]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
@@ -592,16 +592,16 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = fcmp ugt <4 x half> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = fcmp ugt <4 x half> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = fcmp ugt <4 x half> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP100]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP101]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP102]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> zeroinitializer, <4 x i1> [[TMP103]], <4 x i1> zeroinitializer
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> [[TMP76]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> [[TMP77]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> [[TMP70]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> [[TMP71]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index 6727e6b6fb2b4..5e3ec04eff27f 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -120,7 +120,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP52:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP50:%.*]], %[[PRED_LOAD_CONTINUE11]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -188,14 +188,14 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x i64> [ [[TMP35]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP41]], %[[PRED_LOAD_IF10]] ]
; CHECK-NEXT: [[TMP47:%.*]] = phi <4 x i64> [ [[TMP36]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP45]], %[[PRED_LOAD_IF10]] ]
; CHECK-NEXT: [[TMP48:%.*]] = icmp sgt <4 x i64> [[TMP46]], [[TMP47]]
-; CHECK-NEXT: [[TMP49]] = select <4 x i1> [[TMP48]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP49:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP48]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP52]] = select <4 x i1> [[TMP49]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP50]] = or <4 x i1> [[VEC_PHI3]], [[TMP48]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP52:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP49]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP53:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP52]])
; CHECK-NEXT: [[TMP54:%.*]] = add nsw i64 [[TMP53]], -1
; CHECK-NEXT: [[TMP55:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP50]], <4 x i1> [[VEC_PHI3]]
@@ -209,7 +209,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
entry:
br label %loop
-loop: ; preds = %entry, %loop
+loop:
%iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ]
%rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ]
%iv.next = add nsw i64 %iv, -1
@@ -222,6 +222,6 @@ loop: ; preds = %entry, %loop
%exit.cond = icmp ugt i64 %iv, 1
br i1 %exit.cond, label %loop, label %exit
-exit: ; preds = %loop
+exit:
ret i64 %cond
}
>From fe17ce573d5c9613dd0ad2ae8537979679b058f6 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 9 Apr 2026 19:24:59 -0700
Subject: [PATCH 4/4] !Fixup, mask select cond with header mask.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 17 +++------
.../find-last-iv-sinkable-expr.ll | 27 ++++++++------
.../LoopVectorize/iv-select-cmp-decreasing.ll | 36 ++++++++-----------
.../LoopVectorize/iv-select-cmp-fold-tail.ll | 5 ++-
4 files changed, 38 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3de00a5982001..2c5c0c5903097 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5882,9 +5882,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
// propagating poison from inactive lanes in the last vector iteration.
if (HeaderMask) {
VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
- auto *Or = LoopBuilder.createLogicalAnd(HeaderMask,
- FindLastSelect->getOperand(0));
- FindLastSelect->setOperand(0, Or);
+ Cond = LoopBuilder.createLogicalAnd(HeaderMask, Cond);
}
VPInstruction *RdxResult = cast<VPInstruction>(vputils::findRecipe(
@@ -5897,15 +5895,14 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
auto *OrigFindLastSelectR = FindLastSelect->getDefiningRecipe();
VPBuilder LoopBuilder(OrigFindLastSelectR);
DebugLoc DL = OrigFindLastSelectR->getDebugLoc();
- VPValue *SelectCond = OrigFindLastSelectR->getOperand(0);
if (OrigFindLastSelectR->getOperand(1) == PhiR) {
- FindLastSelect = LoopBuilder.createSelect(SelectCond, PhiR,
- IVOfExpressionToSink, DL);
+ FindLastSelect =
+ LoopBuilder.createSelect(Cond, PhiR, IVOfExpressionToSink, DL);
} else {
assert(OrigFindLastSelectR->getOperand(2) == PhiR &&
"PhiR expected as operand 1 or 2");
- FindLastSelect = LoopBuilder.createSelect(
- SelectCond, IVOfExpressionToSink, PhiR, DL);
+ FindLastSelect =
+ LoopBuilder.createSelect(Cond, IVOfExpressionToSink, PhiR, DL);
}
}
@@ -5958,10 +5955,6 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
/*IsInLoop=*/false, FastMathFlags());
auto *OrReduce = MiddleBuilder.createNaryOp(
VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
- // Add predicated select to prevent poison value when folding tail.
- if (HeaderMask)
- OrVal = MiddleBuilder.createSelect(HeaderMask, OrVal, AnyOfPhi, ExitDL);
-
NewRdxResult = MiddleBuilder.createNaryOp(
VPInstruction::ComputeAnyOfResult,
{StartVPV, VectorRegionExitingVal, OrReduce}, {}, ExitDL);
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index 19b624027a81c..f4cbbd23ddc39 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -73,7 +73,8 @@ define i64 @findlast_shl_expr(ptr %a, i64 %n) {
; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -174,7 +175,8 @@ define i64 @findlast_mul_expr(ptr %a, i64 %n) {
; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -392,7 +394,8 @@ define i64 @findlast_or_expr(ptr %a, i64 %n) {
; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1195,8 +1198,8 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; TF-NEXT: br label %[[VECTOR_BODY:.*]]
; TF: [[VECTOR_BODY]]:
; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; TF-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; TF-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
@@ -1213,11 +1216,13 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; TF-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i64 -3
; TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP7]], <4 x i1> [[REVERSE]], <4 x i64> poison)
; TF-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; TF-NEXT: [[TMP8:%.*]] = icmp sle <4 x i64> [[REVERSE6]], [[REVERSE9]]
+; TF-NEXT: [[TMP16:%.*]] = icmp sgt <4 x i64> [[REVERSE6]], [[REVERSE9]]
+; TF-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP16]], <4 x i1> zeroinitializer
; TF-NEXT: [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP8]]
+; TF-NEXT: [[TMP17:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP17]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
+; TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_PHI]], splat (i64 -4)
; TF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; TF: [[MIDDLE_BLOCK]]:
@@ -1321,7 +1326,8 @@ define i64 @findlast_sub_n_expr(ptr %a, i64 %n) {
; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1424,7 +1430,8 @@ define i64 @findlast_sub_iv_as_lhs(ptr %a, i64 %n) {
; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index a53b5c0e560ea..b8ef3b0e464e9 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -326,19 +326,15 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = icmp ugt <4 x i16> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = icmp ugt <4 x i16> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = icmp ugt <4 x i16> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP100]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP101]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP102]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> zeroinitializer, <4 x i1> [[TMP103]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> [[TMP76]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> [[TMP77]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> [[TMP70]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> [[TMP71]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
-; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
+; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
@@ -592,19 +588,15 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[TMP101:%.*]] = fcmp ugt <4 x half> [[TMP51]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP102:%.*]] = fcmp ugt <4 x half> [[TMP75]], [[BROADCAST_SPLAT]]
; IC4VF4-NEXT: [[TMP103:%.*]] = fcmp ugt <4 x half> [[TMP99]], [[BROADCAST_SPLAT]]
-; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP100]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP101]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> splat (i1 true), <4 x i1> [[TMP102]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> zeroinitializer, <4 x i1> [[TMP103]], <4 x i1> zeroinitializer
-; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> [[TMP76]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> [[TMP77]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> [[TMP70]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
-; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> [[TMP71]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP76:%.*]] = select <4 x i1> [[TMP100]], <4 x i16> <i16 11, i16 10, i16 9, i16 8>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
-; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
+; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index 5e3ec04eff27f..343c6a4d6e154 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -190,7 +190,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[TMP48:%.*]] = icmp sgt <4 x i64> [[TMP46]], [[TMP47]]
; CHECK-NEXT: [[TMP49:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP48]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP52]] = select <4 x i1> [[TMP49]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP50]] = or <4 x i1> [[VEC_PHI3]], [[TMP48]]
+; CHECK-NEXT: [[TMP50]] = or <4 x i1> [[VEC_PHI3]], [[TMP49]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -198,8 +198,7 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP53:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP52]])
; CHECK-NEXT: [[TMP54:%.*]] = add nsw i64 [[TMP53]], -1
-; CHECK-NEXT: [[TMP55:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP50]], <4 x i1> [[VEC_PHI3]]
-; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP55]])
+; CHECK-NEXT: [[TMP56:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP50]])
; CHECK-NEXT: [[TMP57:%.*]] = freeze i1 [[TMP56]]
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP57]], i64 [[TMP54]], i64 [[RDX_START]]
; CHECK-NEXT: br label %[[EXIT:.*]]
More information about the llvm-commits
mailing list