[llvm] [LV] Vectorize fmin/fmax reductions (PR #198300)
Nashe Mncube via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 03:50:22 PDT 2026
https://github.com/nasherm updated https://github.com/llvm/llvm-project/pull/198300
>From 006c2b613686f60a2cb7008bb3a153d4b931cd3e Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Mon, 30 Mar 2026 11:02:21 +0100
Subject: [PATCH 1/4] [LV] Vectorize indexed min/max reduction chains
Teach LoopVectorize to recognize min/max reductions whose compare feeds a
parallel select-based def-use chain, enabling vectorization of loops that
reduce both the extremum value and its corresponding index.
Extend the min/max recurrence matching to tolerate these extra select users,
handle floating-point predicates in the multi-use reduction path, and preserve
unsupported cases that do not match the required pattern.
Assisted-by: Codex
Change-Id: Ib729932c552fb1387e25400f1a21a63c69c52d2e
---
llvm/lib/Analysis/IVDescriptors.cpp | 41 +-
.../Vectorize/LoopVectorizationLegality.cpp | 1 +
.../Vectorize/VPlanConstruction.cpp | 16 +-
.../LoopVectorize/select-last-index-fp.ll | 204 +++-
.../select-minmax-multi-def-use.ll | 899 ++++++++++++++++++
5 files changed, 1125 insertions(+), 36 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 9d30928d1751c..0095ab1b50619 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -292,6 +292,19 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
return RecurKind::FMinimumNum;
if (match(V, m_Intrinsic<Intrinsic::maximumnum>(m_Value(A), m_Value(B))))
return RecurKind::FMaximumNum;
+ // A select(fcmp(A, B)) is equivalent to a min/max recurrence
+ // depending on the condition used
+ if (match(V, m_FCmp(m_Value(A), m_Value(B)))) {
+ User *U = V->getUniqueUndroppableUser();
+ if (U && match(U, m_Select(m_Cmp(), m_Value(A), m_Value(B)))) {
+ const auto P = dyn_cast<FCmpInst>(V)->getPredicate();
+ // Determine that the predicate is relational
+ if (P == CmpInst::FCMP_OLT || P == CmpInst::FCMP_ULT)
+ return RecurKind::FMax;
+ if (P == CmpInst::FCMP_OGT || P == CmpInst::FCMP_UGT)
+ return RecurKind::FMin;
+ }
+ }
return RecurKind::None;
};
@@ -355,7 +368,7 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
GetMinMaxRK(U, A, B) == RecurKind::None;
});
if (PhiHasInvalidUses) {
- if (!RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RK) ||
+ if (!RecurrenceDescriptor::isMinMaxRecurrenceKind(RK) ||
!BackedgeValue->hasOneUse())
return {};
return RecurrenceDescriptor(
@@ -367,6 +380,7 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
// Validate chain entries and collect stores from chain entries and
// intermediate ops.
SmallVector<StoreInst *> Stores;
+ SmallPtrSet<Value *, 8> SubChains;
for (Value *V : Chain) {
for (User *U : V->users()) {
if (Chain.contains(U))
@@ -384,10 +398,31 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
Value *A, *B;
if (GetMinMaxRK(I, A, B) != RK)
return {};
+
+ // Results of FCmps should be used by single selects to qualify as valid
+ // intermediate uses of the Phi. These selects must exist as backedge
+ // values to unique def-use chains. These unique chains should be handled
+ // as separate but parralel "sub" min/max recurrences and should not cause
+ // this chain to fail getting the recurrence descriptor.
+ if (auto *FCmpI = dyn_cast<FCmpInst>(I)) {
+ auto *Select = dyn_cast<SelectInst>(FCmpI->getUniqueUndroppableUser());
+ if (!Select || !TheLoop->contains(Select))
+ return {};
+
+ bool SubChainIsValid = any_of(Select->users(), [&](User *US) {
+ auto *SubChain = dyn_cast<PHINode>(US);
+ return SubChain && !Chain.contains(SubChain);
+ });
+
+ if (!SubChainIsValid)
+ return {};
+ SubChains.insert(Select);
+ }
+
for (User *IU : I->users()) {
if (auto *SI = dyn_cast<StoreInst>(IU))
Stores.push_back(SI);
- else if (!Chain.contains(IU))
+ else if (!Chain.contains(IU) && !SubChains.contains(IU))
return {};
}
}
@@ -1090,6 +1125,7 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
+
auto RD = getMinMaxRecurrence(Phi, TheLoop, SE);
if (RD.getRecurrenceKind() != RecurKind::None) {
assert(
@@ -1434,6 +1470,7 @@ bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop,
// Here we only handle FP induction variables.
assert(Phi->getType()->isFloatingPointTy() && "Unexpected Phi type");
+ errs() << "Phi = " << *Phi << '\n';
if (TheLoop->getHeader() != Phi->getParent())
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 214b5cff8a03a..89b42611ed27d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -837,6 +837,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return false;
}
+ // TODO: failing here
RecurrenceDescriptor RedDes;
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
PSE.getSE())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index d6b70068fe7f9..5fc822d630e44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2082,7 +2082,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
// reduction cycle.
RecurKind RdxKind = MinOrMaxPhiR->getRecurrenceKind();
assert(
- RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) &&
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"only min/max recurrences support users outside the reduction chain");
auto *MinOrMaxOp =
@@ -2177,6 +2177,16 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SLE || Pred == CmpInst::ICMP_SLT;
case RecurKind::SMin:
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
+ case RecurKind::FMax:
+ case RecurKind::FMaximum:
+ case RecurKind::FMaxNum:
+ case RecurKind::FMaximumNum:
+ return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT;
+ case RecurKind::FMin:
+ case RecurKind::FMinimum:
+ case RecurKind::FMinNum:
+ case RecurKind::FMinimumNum:
+ return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT;
default:
llvm_unreachable("unhandled recurrence kind");
}
@@ -2240,7 +2250,9 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
VPBuilder B(FindIVRdxResult);
VPValue *MinOrMaxExiting = MinOrMaxResult->getOperand(0);
auto *FinalMinOrMaxCmp =
- B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
+ (RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind))
+ ? B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult)
+ : B.createFCmp(CmpInst::FCMP_OEQ, MinOrMaxExiting, MinOrMaxResult);
VPValue *Sentinel = FindIVCmp->getOperand(1);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index 0ae2141178a0c..389ace0c29570 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -264,21 +264,56 @@ define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -307,21 +342,56 @@ define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -350,21 +420,56 @@ define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -393,21 +498,56 @@ define i64 @test_fmaximumnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximumnum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
new file mode 100644
index 0000000000000..f34de2044d2c2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
@@ -0,0 +1,899 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+declare float @llvm.maxnum.f32(float, float)
+declare double @llvm.maxnum.f64(double, double)
+declare float @llvm.minnum.f32(float, float)
+declare double @llvm.minnum.f64(double, double)
+declare i32 @llvm.smax.i32(i32, i32)
+declare i64 @llvm.smax.i64(i64, i64)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i64 @llvm.smin.i64(i64, i64)
+
+define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f32_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp fast olt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call nnan ninf nsz float @llvm.maxnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f64_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast olt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.maxnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx.lcssa, ptr %out.idx, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast olt double %val.0, %1
+ %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
+ %val.1 = tail call nnan ninf nsz double @llvm.maxnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmin_f32_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast ogt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp fast ogt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call nnan ninf nsz float @llvm.minnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmin_f64_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.minnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx.lcssa, ptr %out.idx, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast ogt double %val.0, %1
+ %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
+ %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i64 @smax_i32_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smax_i32_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], -2147483648
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -2147483648, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi i32 [ -2147483648, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp slt i32 %max.val, %l
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %max.val.next = call i32 @llvm.smax.i32(i32 %max.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smax_i64_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smax_i64_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi i64 [ -9223372036854775808, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr i64, ptr %src, i64 %iv
+ %l = load i64, ptr %gep, align 8
+ %cmp = icmp slt i64 %max.val, %l
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %max.val.next = call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smin_i32_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i32_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 2147483647), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 2147483647
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 2147483647, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %min.val, %l
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smin_i64_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i64_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 9223372036854775807
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 9223372036854775807, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i64 [ 9223372036854775807, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i64, ptr %src, i64 %iv
+ %l = load i64, ptr %gep, align 8
+ %cmp = icmp sgt i64 %min.val, %l
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %min.val.next = call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
+; Unsupported: FP chain lacks the required fast-math flags.
+define void @fmax_f32_val_and_idx_no_nnan(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f32_val_and_idx_no_nnan(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL_0]], [[TMP1]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP1]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp olt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call float @llvm.maxnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Unsupported: compare feeds multiple select chains.
+define void @fmin_f64_val_and_two_idxs_bad(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx0, ptr %out.idx1) {
+; CHECK-LABEL: define void @fmin_f64_val_and_two_idxs_bad(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX0:%.*]], ptr [[OUT_IDX1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX0_1_LCSSA:%.*]] = phi i64 [ [[IDX0_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[IDX1_1_LCSSA:%.*]] = phi i64 [ [[IDX1_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX0_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[IDX1_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX1_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX0_LCSSA]], ptr [[OUT_IDX0]], align 8
+; CHECK-NEXT: store i64 [[IDX1_LCSSA]], ptr [[OUT_IDX1]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX0_0:%.*]] = phi i64 [ [[IDX0_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX1_0:%.*]] = phi i64 [ [[IDX1_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP1]]
+; CHECK-NEXT: [[IDX0_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX0_0]]
+; CHECK-NEXT: [[IDX1_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX1_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP1]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx0.lcssa = phi i64 [ 0, %entry ], [ %idx0.1, %for.body ]
+ %idx1.lcssa = phi i64 [ 0, %entry ], [ %idx1.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx0.lcssa, ptr %out.idx0, align 8
+ store i64 %idx1.lcssa, ptr %out.idx1, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx0.0 = phi i64 [ %idx0.1, %for.body ], [ 0, %entry ]
+ %idx1.0 = phi i64 [ %idx1.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast ogt double %val.0, %1
+ %idx0.1 = select i1 %cmp, i64 %iv, i64 %idx0.0
+ %idx1.1 = select i1 %cmp, i64 %iv, i64 %idx1.0
+ %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Unsupported: selected value is not the induction value.
+define i64 @smin_i32_idx_shifted_bad(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i32_idx_shifted_bad(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[IV_PLUS1:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_PLUS1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %min.val, %l
+ %iv.plus1 = add nuw nsw i64 %iv, 1
+ %min.idx.next = select i1 %cmp, i64 %iv.plus1, i64 %min.idx
+ %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
>From 049a59a095457ffcb488c60ed6be8a91086b137c Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Mon, 18 May 2026 14:44:15 +0100
Subject: [PATCH 2/4] Remove comments
Change-Id: Id4ee04836e3b49f3cd02c046f3982cf8265bc66e
---
llvm/lib/Analysis/IVDescriptors.cpp | 2 --
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 1 -
2 files changed, 3 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 0095ab1b50619..566c1b229b89f 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1125,7 +1125,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
-
auto RD = getMinMaxRecurrence(Phi, TheLoop, SE);
if (RD.getRecurrenceKind() != RecurKind::None) {
assert(
@@ -1470,7 +1469,6 @@ bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop,
// Here we only handle FP induction variables.
assert(Phi->getType()->isFloatingPointTy() && "Unexpected Phi type");
- errs() << "Phi = " << *Phi << '\n';
if (TheLoop->getHeader() != Phi->getParent())
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 89b42611ed27d..214b5cff8a03a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -837,7 +837,6 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return false;
}
- // TODO: failing here
RecurrenceDescriptor RedDes;
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
PSE.getSE())) {
>From 8853725f621c0937559a7a839ed7a97c559b89dc Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Wed, 20 May 2026 09:43:54 +0100
Subject: [PATCH 3/4] Breakup patch
Responding to review comments by breaking up
the patch. Work in this patch now only deals with
adding support to the vectorizer for dealing with
max/min reductions.
Change-Id: I84681eb9518e5d526d6f705da9516656fbf57be4
---
llvm/lib/Analysis/IVDescriptors.cpp | 37 +--
.../LoopVectorize/select-first-index-fp.ll | 204 +++++++++++---
.../select-minmax-multi-def-use.ll | 252 +++---------------
3 files changed, 217 insertions(+), 276 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 566c1b229b89f..bd45d250db4e8 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -292,19 +292,6 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
return RecurKind::FMinimumNum;
if (match(V, m_Intrinsic<Intrinsic::maximumnum>(m_Value(A), m_Value(B))))
return RecurKind::FMaximumNum;
- // A select(fcmp(A, B)) is equivalent to a min/max recurrence
- // depending on the condition used
- if (match(V, m_FCmp(m_Value(A), m_Value(B)))) {
- User *U = V->getUniqueUndroppableUser();
- if (U && match(U, m_Select(m_Cmp(), m_Value(A), m_Value(B)))) {
- const auto P = dyn_cast<FCmpInst>(V)->getPredicate();
- // Determine that the predicate is relational
- if (P == CmpInst::FCMP_OLT || P == CmpInst::FCMP_ULT)
- return RecurKind::FMax;
- if (P == CmpInst::FCMP_OGT || P == CmpInst::FCMP_UGT)
- return RecurKind::FMin;
- }
- }
return RecurKind::None;
};
@@ -380,7 +367,6 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
// Validate chain entries and collect stores from chain entries and
// intermediate ops.
SmallVector<StoreInst *> Stores;
- SmallPtrSet<Value *, 8> SubChains;
for (Value *V : Chain) {
for (User *U : V->users()) {
if (Chain.contains(U))
@@ -398,31 +384,10 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
Value *A, *B;
if (GetMinMaxRK(I, A, B) != RK)
return {};
-
- // Results of FCmps should be used by single selects to qualify as valid
- // intermediate uses of the Phi. These selects must exist as backedge
- // values to unique def-use chains. These unique chains should be handled
- // as separate but parralel "sub" min/max recurrences and should not cause
- // this chain to fail getting the recurrence descriptor.
- if (auto *FCmpI = dyn_cast<FCmpInst>(I)) {
- auto *Select = dyn_cast<SelectInst>(FCmpI->getUniqueUndroppableUser());
- if (!Select || !TheLoop->contains(Select))
- return {};
-
- bool SubChainIsValid = any_of(Select->users(), [&](User *US) {
- auto *SubChain = dyn_cast<PHINode>(US);
- return SubChain && !Chain.contains(SubChain);
- });
-
- if (!SubChainIsValid)
- return {};
- SubChains.insert(Select);
- }
-
for (User *IU : I->users()) {
if (auto *SI = dyn_cast<StoreInst>(IU))
Stores.push_back(SI);
- else if (!Chain.contains(IU) && !SubChains.contains(IU))
+ else if (!Chain.contains(IU))
return {};
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
index 4ab5eaee6b749..04916a11db6b0 100644
--- a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
@@ -355,21 +355,56 @@ define i64 @test_fminimum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -398,21 +433,56 @@ define i64 @test_fmaximumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -441,21 +511,56 @@ define i64 @test_fminimumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -484,21 +589,56 @@ define i64 @test_fmaximum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
index f34de2044d2c2..a583306259520 100644
--- a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
@@ -16,54 +16,12 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -72,10 +30,10 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
@@ -83,7 +41,7 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load float, ptr %src, align 4
@@ -118,52 +76,12 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast olt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.maxnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -172,10 +90,10 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
@@ -183,7 +101,7 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load double, ptr %src, align 8
@@ -218,54 +136,12 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast ogt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -274,10 +150,10 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
@@ -285,7 +161,7 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load float, ptr %src, align 4
@@ -320,52 +196,12 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.minnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -374,10 +210,10 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
@@ -385,7 +221,7 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load double, ptr %src, align 8
@@ -437,7 +273,7 @@ define i64 @smax_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
@@ -465,7 +301,7 @@ define i64 @smax_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -514,7 +350,7 @@ define i64 @smax_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
@@ -542,7 +378,7 @@ define i64 @smax_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -591,7 +427,7 @@ define i64 @smin_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
@@ -619,7 +455,7 @@ define i64 @smin_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -668,7 +504,7 @@ define i64 @smin_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
@@ -696,7 +532,7 @@ define i64 @smin_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
>From 1e17d20dfe76e9fd02e57023a6a10c4ead73be9a Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Wed, 20 May 2026 10:27:19 +0100
Subject: [PATCH 4/4] Remove test
Removing the select-minmax-multi-def-use.ll as it no
longer tests the changes added by the broken up patch.
Change-Id: I7235be7f682d747f71a35ff921df72ae66c27221
---
.../select-minmax-multi-def-use.ll | 735 ------------------
1 file changed, 735 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
deleted file mode 100644
index a583306259520..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
+++ /dev/null
@@ -1,735 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-
-declare float @llvm.maxnum.f32(float, float)
-declare double @llvm.maxnum.f64(double, double)
-declare float @llvm.minnum.f32(float, float)
-declare double @llvm.minnum.f64(double, double)
-declare i32 @llvm.smax.i32(i32, i32)
-declare i64 @llvm.smax.i64(i64, i64)
-declare i32 @llvm.smin.i32(i32, i32)
-declare i64 @llvm.smin.i64(i64, i64)
-
-define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f32_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp fast olt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call nnan ninf nsz float @llvm.maxnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f64_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx.lcssa, ptr %out.idx, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast olt double %val.0, %1
- %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
- %val.1 = tail call nnan ninf nsz double @llvm.maxnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmin_f32_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp fast ogt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call nnan ninf nsz float @llvm.minnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmin_f64_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx.lcssa, ptr %out.idx, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast ogt double %val.0, %1
- %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
- %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define i64 @smax_i32_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smax_i32_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], -2147483648
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -2147483648, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
- %max.val = phi i32 [ -2147483648, %entry ], [ %max.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp slt i32 %max.val, %l
- %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
- %max.val.next = call i32 @llvm.smax.i32(i32 %max.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %max.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smax_i64_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smax_i64_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
- %max.val = phi i64 [ -9223372036854775808, %entry ], [ %max.val.next, %loop ]
- %gep = getelementptr i64, ptr %src, i64 %iv
- %l = load i64, ptr %gep, align 8
- %cmp = icmp slt i64 %max.val, %l
- %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
- %max.val.next = call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %max.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smin_i32_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i32_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 2147483647), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 2147483647
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 2147483647, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp sgt i32 %min.val, %l
- %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
- %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smin_i64_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i64_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 9223372036854775807
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 9223372036854775807, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i64 [ 9223372036854775807, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i64, ptr %src, i64 %iv
- %l = load i64, ptr %gep, align 8
- %cmp = icmp sgt i64 %min.val, %l
- %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
- %min.val.next = call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
-
-; Unsupported: FP chain lacks the required fast-math flags.
-define void @fmax_f32_val_and_idx_no_nnan(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f32_val_and_idx_no_nnan(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL_0]], [[TMP1]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP1]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp olt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call float @llvm.maxnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-; Unsupported: compare feeds multiple select chains.
-define void @fmin_f64_val_and_two_idxs_bad(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx0, ptr %out.idx1) {
-; CHECK-LABEL: define void @fmin_f64_val_and_two_idxs_bad(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX0:%.*]], ptr [[OUT_IDX1:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX0_1_LCSSA:%.*]] = phi i64 [ [[IDX0_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[IDX1_1_LCSSA:%.*]] = phi i64 [ [[IDX1_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX0_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[IDX1_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX1_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX0_LCSSA]], ptr [[OUT_IDX0]], align 8
-; CHECK-NEXT: store i64 [[IDX1_LCSSA]], ptr [[OUT_IDX1]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX0_0:%.*]] = phi i64 [ [[IDX0_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX1_0:%.*]] = phi i64 [ [[IDX1_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP1]]
-; CHECK-NEXT: [[IDX0_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX0_0]]
-; CHECK-NEXT: [[IDX1_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX1_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP1]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx0.lcssa = phi i64 [ 0, %entry ], [ %idx0.1, %for.body ]
- %idx1.lcssa = phi i64 [ 0, %entry ], [ %idx1.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx0.lcssa, ptr %out.idx0, align 8
- store i64 %idx1.lcssa, ptr %out.idx1, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx0.0 = phi i64 [ %idx0.1, %for.body ], [ 0, %entry ]
- %idx1.0 = phi i64 [ %idx1.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast ogt double %val.0, %1
- %idx0.1 = select i1 %cmp, i64 %iv, i64 %idx0.0
- %idx1.1 = select i1 %cmp, i64 %iv, i64 %idx1.0
- %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-; Unsupported: selected value is not the induction value.
-define i64 @smin_i32_idx_shifted_bad(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i32_idx_shifted_bad(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[IV_PLUS1:%.*]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_PLUS1]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp sgt i32 %min.val, %l
- %iv.plus1 = add nuw nsw i64 %iv, 1
- %min.idx.next = select i1 %cmp, i64 %iv.plus1, i64 %min.idx
- %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
More information about the llvm-commits
mailing list