[llvm] [LV] Vectorize fmin/fmax reductions (PR #198300)
Nashe Mncube via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 04:49:20 PDT 2026
https://github.com/nasherm updated https://github.com/llvm/llvm-project/pull/198300
>From 48d70e955dac5840c25cbdb79ca832cd089183f7 Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Mon, 30 Mar 2026 11:02:21 +0100
Subject: [PATCH 01/13] [LV] Vectorize indexed min/max reduction chains
Teach LoopVectorize to recognize min/max reductions whose compare feeds a
parallel select-based def-use chain, enabling vectorization of loops that
reduce both the extremum value and its corresponding index.
Extend the min/max recurrence matching to tolerate these extra select users,
handle floating-point predicates in the multi-use reduction path, and preserve
unsupported cases that do not match the required pattern.
Assisted-by: Codex
Change-Id: Ib729932c552fb1387e25400f1a21a63c69c52d2e
---
llvm/lib/Analysis/IVDescriptors.cpp | 41 +-
.../Vectorize/LoopVectorizationLegality.cpp | 1 +
.../Vectorize/VPlanConstruction.cpp | 16 +-
.../LoopVectorize/select-last-index-fp.ll | 204 +++-
.../select-minmax-multi-def-use.ll | 899 ++++++++++++++++++
5 files changed, 1125 insertions(+), 36 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 800b64e9a29af..26908ba227db1 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -292,6 +292,19 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
return RecurKind::FMinimumNum;
if (match(V, m_Intrinsic<Intrinsic::maximumnum>(m_Value(A), m_Value(B))))
return RecurKind::FMaximumNum;
+ // A select(fcmp(A, B)) is equivalent to a min/max recurrence
+ // depending on the condition used
+ if (match(V, m_FCmp(m_Value(A), m_Value(B)))) {
+ User *U = V->getUniqueUndroppableUser();
+ if (U && match(U, m_Select(m_Cmp(), m_Value(A), m_Value(B)))) {
+ const auto P = dyn_cast<FCmpInst>(V)->getPredicate();
+ // Determine that the predicate is relational
+ if (P == CmpInst::FCMP_OLT || P == CmpInst::FCMP_ULT)
+ return RecurKind::FMax;
+ if (P == CmpInst::FCMP_OGT || P == CmpInst::FCMP_UGT)
+ return RecurKind::FMin;
+ }
+ }
return RecurKind::None;
};
@@ -355,7 +368,7 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
GetMinMaxRK(U, A, B) == RecurKind::None;
});
if (PhiHasInvalidUses) {
- if (!RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RK) ||
+ if (!RecurrenceDescriptor::isMinMaxRecurrenceKind(RK) ||
!BackedgeValue->hasOneUse())
return {};
return RecurrenceDescriptor(
@@ -367,6 +380,7 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
// Validate chain entries and collect stores from chain entries and
// intermediate ops.
SmallVector<StoreInst *> Stores;
+ SmallPtrSet<Value *, 8> SubChains;
for (Value *V : Chain) {
for (User *U : V->users()) {
if (Chain.contains(U))
@@ -384,10 +398,31 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
Value *A, *B;
if (GetMinMaxRK(I, A, B) != RK)
return {};
+
+ // Results of FCmps should be used by single selects to qualify as valid
+ // intermediate uses of the Phi. These selects must exist as backedge
+ // values to unique def-use chains. These unique chains should be handled
+ // as separate but parralel "sub" min/max recurrences and should not cause
+ // this chain to fail getting the recurrence descriptor.
+ if (auto *FCmpI = dyn_cast<FCmpInst>(I)) {
+ auto *Select = dyn_cast<SelectInst>(FCmpI->getUniqueUndroppableUser());
+ if (!Select || !TheLoop->contains(Select))
+ return {};
+
+ bool SubChainIsValid = any_of(Select->users(), [&](User *US) {
+ auto *SubChain = dyn_cast<PHINode>(US);
+ return SubChain && !Chain.contains(SubChain);
+ });
+
+ if (!SubChainIsValid)
+ return {};
+ SubChains.insert(Select);
+ }
+
for (User *IU : I->users()) {
if (auto *SI = dyn_cast<StoreInst>(IU))
Stores.push_back(SI);
- else if (!Chain.contains(IU))
+ else if (!Chain.contains(IU) && !SubChains.contains(IU))
return {};
}
}
@@ -1090,6 +1125,7 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
+
auto RD = getMinMaxRecurrence(Phi, TheLoop, SE);
if (RD.getRecurrenceKind() != RecurKind::None) {
assert(
@@ -1434,6 +1470,7 @@ bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop,
// Here we only handle FP induction variables.
assert(Phi->getType()->isFloatingPointTy() && "Unexpected Phi type");
+ errs() << "Phi = " << *Phi << '\n';
if (TheLoop->getHeader() != Phi->getParent())
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 6a5419dfbaccf..d665da58bc43a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -872,6 +872,7 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return false;
}
+ // TODO: failing here
RecurrenceDescriptor RedDes;
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
PSE.getSE())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4c09e10fe6ea0..f1fe9b5c1dfd9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2083,7 +2083,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
// reduction cycle.
RecurKind RdxKind = MinOrMaxPhiR->getRecurrenceKind();
assert(
- RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) &&
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) &&
"only min/max recurrences support users outside the reduction chain");
auto *MinOrMaxOp =
@@ -2178,6 +2178,16 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SLE || Pred == CmpInst::ICMP_SLT;
case RecurKind::SMin:
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
+ case RecurKind::FMax:
+ case RecurKind::FMaximum:
+ case RecurKind::FMaxNum:
+ case RecurKind::FMaximumNum:
+ return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT;
+ case RecurKind::FMin:
+ case RecurKind::FMinimum:
+ case RecurKind::FMinNum:
+ case RecurKind::FMinimumNum:
+ return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT;
default:
llvm_unreachable("unhandled recurrence kind");
}
@@ -2241,7 +2251,9 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
VPBuilder B(FindIVRdxResult);
VPValue *MinOrMaxExiting = MinOrMaxResult->getOperand(0);
auto *FinalMinOrMaxCmp =
- B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
+ (RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind))
+ ? B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult)
+ : B.createFCmp(CmpInst::FCMP_OEQ, MinOrMaxExiting, MinOrMaxResult);
VPValue *Sentinel = FindIVCmp->getOperand(1);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index 0ae2141178a0c..389ace0c29570 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -264,21 +264,56 @@ define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -307,21 +342,56 @@ define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -350,21 +420,56 @@ define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -393,21 +498,56 @@ define i64 @test_fmaximumnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximumnum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
new file mode 100644
index 0000000000000..f34de2044d2c2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
@@ -0,0 +1,899 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+declare float @llvm.maxnum.f32(float, float)
+declare double @llvm.maxnum.f64(double, double)
+declare float @llvm.minnum.f32(float, float)
+declare double @llvm.minnum.f64(double, double)
+declare i32 @llvm.smax.i32(i32, i32)
+declare i64 @llvm.smax.i64(i64, i64)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i64 @llvm.smin.i64(i64, i64)
+
+define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f32_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp fast olt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call nnan ninf nsz float @llvm.maxnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f64_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast olt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.maxnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx.lcssa, ptr %out.idx, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast olt double %val.0, %1
+ %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
+ %val.1 = tail call nnan ninf nsz double @llvm.maxnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmin_f32_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast ogt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp fast ogt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call nnan ninf nsz float @llvm.minnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmin_f64_val_and_idx(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.minnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx.lcssa, ptr %out.idx, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast ogt double %val.0, %1
+ %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
+ %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i64 @smax_i32_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smax_i32_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], -2147483648
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -2147483648, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi i32 [ -2147483648, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp slt i32 %max.val, %l
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %max.val.next = call i32 @llvm.smax.i32(i32 %max.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smax_i64_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smax_i64_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi i64 [ -9223372036854775808, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr i64, ptr %src, i64 %iv
+ %l = load i64, ptr %gep, align 8
+ %cmp = icmp slt i64 %max.val, %l
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %max.val.next = call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smin_i32_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i32_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 2147483647), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 2147483647
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 2147483647, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %min.val, %l
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
+define i64 @smin_i64_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i64_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 9223372036854775807
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 9223372036854775807, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i64 [ 9223372036854775807, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i64, ptr %src, i64 %iv
+ %l = load i64, ptr %gep, align 8
+ %cmp = icmp sgt i64 %min.val, %l
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %min.val.next = call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
+; Unsupported: FP chain lacks the required fast-math flags.
+define void @fmax_f32_val_and_idx_no_nnan(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
+; CHECK-LABEL: define void @fmax_f32_val_and_idx_no_nnan(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
+; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC_NEXT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL_0]], [[TMP1]]
+; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP1]])
+; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
+;
+entry:
+ %0 = load float, ptr %src, align 4
+ %cmp8 = icmp ugt i32 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
+ %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
+ store float %val.lcssa, ptr %out.val, align 4
+ store i32 %idx.lcssa, ptr %out.idx, align 4
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
+ %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
+ %1 = load float, ptr %src.next, align 4
+ %cmp = fcmp olt float %val.0, %1
+ %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
+ %val.1 = tail call float @llvm.maxnum.f32(float %val.0, float %1)
+ %inc = add nuw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Unsupported: compare feeds multiple select chains.
+define void @fmin_f64_val_and_two_idxs_bad(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx0, ptr %out.idx1) {
+; CHECK-LABEL: define void @fmin_f64_val_and_two_idxs_bad(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX0:%.*]], ptr [[OUT_IDX1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
+; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX0_1_LCSSA:%.*]] = phi i64 [ [[IDX0_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[IDX1_1_LCSSA:%.*]] = phi i64 [ [[IDX1_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[IDX0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX0_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[IDX1_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX1_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
+; CHECK-NEXT: store i64 [[IDX0_LCSSA]], ptr [[OUT_IDX0]], align 8
+; CHECK-NEXT: store i64 [[IDX1_LCSSA]], ptr [[OUT_IDX1]], align 8
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX0_0:%.*]] = phi i64 [ [[IDX0_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[IDX1_0:%.*]] = phi i64 [ [[IDX1_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[SRC_NEXT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP1]]
+; CHECK-NEXT: [[IDX0_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX0_0]]
+; CHECK-NEXT: [[IDX1_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX1_0]]
+; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP1]])
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
+;
+entry:
+ %0 = load double, ptr %src, align 8
+ %cmp8 = icmp ugt i64 %n, 1
+ br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %idx0.lcssa = phi i64 [ 0, %entry ], [ %idx0.1, %for.body ]
+ %idx1.lcssa = phi i64 [ 0, %entry ], [ %idx1.1, %for.body ]
+ %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
+ store double %val.lcssa, ptr %out.val, align 8
+ store i64 %idx0.lcssa, ptr %out.idx0, align 8
+ store i64 %idx1.lcssa, ptr %out.idx1, align 8
+ ret void
+
+for.body:
+ %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
+ %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
+ %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
+ %idx0.0 = phi i64 [ %idx0.1, %for.body ], [ 0, %entry ]
+ %idx1.0 = phi i64 [ %idx1.1, %for.body ], [ 0, %entry ]
+ %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
+ %1 = load double, ptr %src.next, align 8
+ %cmp = fcmp fast ogt double %val.0, %1
+ %idx0.1 = select i1 %cmp, i64 %iv, i64 %idx0.0
+ %idx1.1 = select i1 %cmp, i64 %iv, i64 %idx1.0
+ %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
+ %inc = add nuw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, %n
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Unsupported: selected value is not the induction value.
+define i64 @smin_i32_idx_shifted_bad(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @smin_i32_idx_shifted_bad(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[IV_PLUS1:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_PLUS1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr i32, ptr %src, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %min.val, %l
+ %iv.plus1 = add nuw nsw i64 %iv, 1
+ %min.idx.next = select i1 %cmp, i64 %iv.plus1, i64 %min.idx
+ %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
>From 9f002602350b51b40c091ab6afea99c09b59a261 Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Mon, 18 May 2026 14:44:15 +0100
Subject: [PATCH 02/13] Remove comments
Change-Id: Id4ee04836e3b49f3cd02c046f3982cf8265bc66e
---
llvm/lib/Analysis/IVDescriptors.cpp | 2 --
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 1 -
2 files changed, 3 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 26908ba227db1..9a0b0de6bbfeb 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1125,7 +1125,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
return true;
}
-
auto RD = getMinMaxRecurrence(Phi, TheLoop, SE);
if (RD.getRecurrenceKind() != RecurKind::None) {
assert(
@@ -1470,7 +1469,6 @@ bool InductionDescriptor::isFPInductionPHI(PHINode *Phi, const Loop *TheLoop,
// Here we only handle FP induction variables.
assert(Phi->getType()->isFloatingPointTy() && "Unexpected Phi type");
- errs() << "Phi = " << *Phi << '\n';
if (TheLoop->getHeader() != Phi->getParent())
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index d665da58bc43a..6a5419dfbaccf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -872,7 +872,6 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return false;
}
- // TODO: failing here
RecurrenceDescriptor RedDes;
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
PSE.getSE())) {
>From f325e5eb7df49b9fc4eed6ba0f7f10265a90f2f5 Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Wed, 20 May 2026 09:43:54 +0100
Subject: [PATCH 03/13] Breakup patch
Responding to review comments by breaking up
the patch. Work in this patch now only deals with
adding support to the vectorizer for dealing with
max/min reductions.
Change-Id: I84681eb9518e5d526d6f705da9516656fbf57be4
---
llvm/lib/Analysis/IVDescriptors.cpp | 37 +--
.../LoopVectorize/select-first-index-fp.ll | 204 +++++++++++---
.../select-minmax-multi-def-use.ll | 252 +++---------------
3 files changed, 217 insertions(+), 276 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 9a0b0de6bbfeb..47b5bf87596d5 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -292,19 +292,6 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
return RecurKind::FMinimumNum;
if (match(V, m_Intrinsic<Intrinsic::maximumnum>(m_Value(A), m_Value(B))))
return RecurKind::FMaximumNum;
- // A select(fcmp(A, B)) is equivalent to a min/max recurrence
- // depending on the condition used
- if (match(V, m_FCmp(m_Value(A), m_Value(B)))) {
- User *U = V->getUniqueUndroppableUser();
- if (U && match(U, m_Select(m_Cmp(), m_Value(A), m_Value(B)))) {
- const auto P = dyn_cast<FCmpInst>(V)->getPredicate();
- // Determine that the predicate is relational
- if (P == CmpInst::FCMP_OLT || P == CmpInst::FCMP_ULT)
- return RecurKind::FMax;
- if (P == CmpInst::FCMP_OGT || P == CmpInst::FCMP_UGT)
- return RecurKind::FMin;
- }
- }
return RecurKind::None;
};
@@ -380,7 +367,6 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
// Validate chain entries and collect stores from chain entries and
// intermediate ops.
SmallVector<StoreInst *> Stores;
- SmallPtrSet<Value *, 8> SubChains;
for (Value *V : Chain) {
for (User *U : V->users()) {
if (Chain.contains(U))
@@ -398,31 +384,10 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
Value *A, *B;
if (GetMinMaxRK(I, A, B) != RK)
return {};
-
- // Results of FCmps should be used by single selects to qualify as valid
- // intermediate uses of the Phi. These selects must exist as backedge
- // values to unique def-use chains. These unique chains should be handled
- // as separate but parralel "sub" min/max recurrences and should not cause
- // this chain to fail getting the recurrence descriptor.
- if (auto *FCmpI = dyn_cast<FCmpInst>(I)) {
- auto *Select = dyn_cast<SelectInst>(FCmpI->getUniqueUndroppableUser());
- if (!Select || !TheLoop->contains(Select))
- return {};
-
- bool SubChainIsValid = any_of(Select->users(), [&](User *US) {
- auto *SubChain = dyn_cast<PHINode>(US);
- return SubChain && !Chain.contains(SubChain);
- });
-
- if (!SubChainIsValid)
- return {};
- SubChains.insert(Select);
- }
-
for (User *IU : I->users()) {
if (auto *SI = dyn_cast<StoreInst>(IU))
Stores.push_back(SI);
- else if (!Chain.contains(IU) && !SubChains.contains(IU))
+ else if (!Chain.contains(IU))
return {};
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
index 4ab5eaee6b749..04916a11db6b0 100644
--- a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
@@ -355,21 +355,56 @@ define i64 @test_fminimum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -398,21 +433,56 @@ define i64 @test_fmaximumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -441,21 +511,56 @@ define i64 @test_fminimumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -484,21 +589,56 @@ define i64 @test_fmaximum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
index f34de2044d2c2..a583306259520 100644
--- a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
@@ -16,54 +16,12 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -72,10 +30,10 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
@@ -83,7 +41,7 @@ define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load float, ptr %src, align 4
@@ -118,52 +76,12 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast olt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.maxnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmax.v4f64(<4 x double> [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -172,10 +90,10 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
@@ -183,7 +101,7 @@ define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load double, ptr %src, align 8
@@ -218,54 +136,12 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast ogt <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP11]] = call nnan ninf nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = icmp ne i32 [[TMP14]], 0
-; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -274,10 +150,10 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
@@ -285,7 +161,7 @@ define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load float, ptr %src, align 4
@@ -320,52 +196,12 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[NEXT_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8]] = select <4 x i1> [[TMP7]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP9]] = call nnan ninf nsz <4 x double> @llvm.minnum.v4f64(<4 x double> [[VEC_PHI]], <4 x double> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call nnan ninf nsz double @llvm.vector.reduce.fmin.v4f64(<4 x double> [[TMP9]])
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP8]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], 0
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
@@ -374,10 +210,10 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
@@ -385,7 +221,7 @@ define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %
; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
;
entry:
%0 = load double, ptr %src, align 8
@@ -437,7 +273,7 @@ define i64 @smax_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
@@ -465,7 +301,7 @@ define i64 @smax_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -514,7 +350,7 @@ define i64 @smax_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
@@ -542,7 +378,7 @@ define i64 @smax_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -591,7 +427,7 @@ define i64 @smin_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
@@ -619,7 +455,7 @@ define i64 @smin_i32_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -668,7 +504,7 @@ define i64 @smin_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
@@ -696,7 +532,7 @@ define i64 @smin_i64_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
>From 74896f84fa87e13192367256e430b8d073c83e5c Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Wed, 20 May 2026 10:27:19 +0100
Subject: [PATCH 04/13] Remove test
Removing the select-minmax-multi-def-use.ll as it no
longer tests the changes added by the broken up patch.
Change-Id: I7235be7f682d747f71a35ff921df72ae66c27221
---
.../select-minmax-multi-def-use.ll | 735 ------------------
1 file changed, 735 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
diff --git a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll b/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
deleted file mode 100644
index a583306259520..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/select-minmax-multi-def-use.ll
+++ /dev/null
@@ -1,735 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-
-declare float @llvm.maxnum.f32(float, float)
-declare double @llvm.maxnum.f64(double, double)
-declare float @llvm.minnum.f32(float, float)
-declare double @llvm.minnum.f64(double, double)
-declare i32 @llvm.smax.i32(i32, i32)
-declare i64 @llvm.smax.i64(i64, i64)
-declare i32 @llvm.smin.i32(i32, i32)
-declare i64 @llvm.smin.i64(i64, i64)
-
-define void @fmax_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f32_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt float [[VAL_0]], [[TMP17]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP17]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp fast olt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call nnan ninf nsz float @llvm.maxnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmax_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f64_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[VAL_0]], [[TMP15]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.maxnum.f64(double [[VAL_0]], double [[TMP15]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx.lcssa, ptr %out.idx, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast olt double %val.0, %1
- %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
- %val.1 = tail call nnan ninf nsz double @llvm.maxnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmin_f32_val_and_idx(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmin_f32_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt float [[VAL_0]], [[TMP17]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz float @llvm.minnum.f32(float [[VAL_0]], float [[TMP17]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp fast ogt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call nnan ninf nsz float @llvm.minnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define void @fmin_f64_val_and_idx(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmin_f64_val_and_idx(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[SCALAR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i64 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i64 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP15]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP15]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i64 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx.lcssa, ptr %out.idx, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i64 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast ogt double %val.0, %1
- %idx.1 = select i1 %cmp, i64 %iv, i64 %idx.0
- %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-define i64 @smax_i32_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smax_i32_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], -2147483648
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -2147483648, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
- %max.val = phi i32 [ -2147483648, %entry ], [ %max.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp slt i32 %max.val, %l
- %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
- %max.val.next = call i32 @llvm.smax.i32(i32 %max.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %max.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smax_i64_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smax_i64_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
- %max.val = phi i64 [ -9223372036854775808, %entry ], [ %max.val.next, %loop ]
- %gep = getelementptr i64, ptr %src, i64 %iv
- %l = load i64, ptr %gep, align 8
- %cmp = icmp slt i64 %max.val, %l
- %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
- %max.val.next = call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %max.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smin_i32_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i32_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 2147483647), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI1]], <4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 2147483647
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 2147483647, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp sgt i32 %min.val, %l
- %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
- %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
-
-define i64 @smin_i64_idx(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i64_idx(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP2]], <4 x i64> splat (i64 -1)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 9223372036854775807
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 0, i64 [[TMP8]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 9223372036854775807, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i64 [ 9223372036854775807, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i64, ptr %src, i64 %iv
- %l = load i64, ptr %gep, align 8
- %cmp = icmp sgt i64 %min.val, %l
- %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
- %min.val.next = call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
-
-; Unsupported: FP chain lacks the required fast-math flags.
-define void @fmax_f32_val_and_idx_no_nnan(ptr readonly %src, i32 %n, ptr %out.val, ptr %out.idx) {
-; CHECK-LABEL: define void @fmax_f32_val_and_idx_no_nnan(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i32 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i32 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX_1_LCSSA:%.*]] = phi i32 [ [[IDX_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi float [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi float [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store float [[VAL_LCSSA]], ptr [[OUT_VAL]], align 4
-; CHECK-NEXT: store i32 [[IDX_LCSSA]], ptr [[OUT_IDX]], align 4
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi float [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX_0:%.*]] = phi i32 [ [[IDX_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 4
-; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC_NEXT]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL_0]], [[TMP1]]
-; CHECK-NEXT: [[IDX_1]] = select i1 [[CMP]], i32 [[IV]], i32 [[IDX_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call float @llvm.maxnum.f32(float [[VAL_0]], float [[TMP1]])
-; CHECK-NEXT: [[INC]] = add nuw i32 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load float, ptr %src, align 4
- %cmp8 = icmp ugt i32 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx.lcssa = phi i32 [ 0, %entry ], [ %idx.1, %for.body ]
- %val.lcssa = phi float [ %0, %entry ], [ %val.1, %for.body ]
- store float %val.lcssa, ptr %out.val, align 4
- store i32 %idx.lcssa, ptr %out.idx, align 4
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i32 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi float [ %val.1, %for.body ], [ %0, %entry ]
- %idx.0 = phi i32 [ %idx.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 4
- %1 = load float, ptr %src.next, align 4
- %cmp = fcmp olt float %val.0, %1
- %idx.1 = select i1 %cmp, i32 %iv, i32 %idx.0
- %val.1 = tail call float @llvm.maxnum.f32(float %val.0, float %1)
- %inc = add nuw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-; Unsupported: compare feeds multiple select chains.
-define void @fmin_f64_val_and_two_idxs_bad(ptr readonly %src, i64 %n, ptr %out.val, ptr %out.idx0, ptr %out.idx1) {
-; CHECK-LABEL: define void @fmin_f64_val_and_two_idxs_bad(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], i64 [[N:%.*]], ptr [[OUT_VAL:%.*]], ptr [[OUT_IDX0:%.*]], ptr [[OUT_IDX1:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[SRC]], align 8
-; CHECK-NEXT: [[CMP8:%.*]] = icmp ugt i64 [[N]], 1
-; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
-; CHECK-NEXT: [[IDX0_1_LCSSA:%.*]] = phi i64 [ [[IDX0_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[IDX1_1_LCSSA:%.*]] = phi i64 [ [[IDX1_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VAL_1_LCSSA:%.*]] = phi double [ [[VAL_1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[IDX0_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX0_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[IDX1_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX1_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi double [ [[TMP0]], %[[ENTRY]] ], [ [[VAL_1_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ]
-; CHECK-NEXT: store double [[VAL_LCSSA]], ptr [[OUT_VAL]], align 8
-; CHECK-NEXT: store i64 [[IDX0_LCSSA]], ptr [[OUT_IDX0]], align 8
-; CHECK-NEXT: store i64 [[IDX1_LCSSA]], ptr [[OUT_IDX1]], align 8
-; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[VAL_0:%.*]] = phi double [ [[VAL_1]], %[[FOR_BODY]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX0_0:%.*]] = phi i64 [ [[IDX0_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[IDX1_0:%.*]] = phi i64 [ [[IDX1_1]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[SRC_NEXT]] = getelementptr inbounds nuw i8, ptr [[SRC_PHI]], i64 8
-; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[SRC_NEXT]], align 8
-; CHECK-NEXT: [[CMP:%.*]] = fcmp fast ogt double [[VAL_0]], [[TMP1]]
-; CHECK-NEXT: [[IDX0_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX0_0]]
-; CHECK-NEXT: [[IDX1_1]] = select i1 [[CMP]], i64 [[IV]], i64 [[IDX1_0]]
-; CHECK-NEXT: [[VAL_1]] = tail call nnan ninf nsz double @llvm.minnum.f64(double [[VAL_0]], double [[TMP1]])
-; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]]
-;
-entry:
- %0 = load double, ptr %src, align 8
- %cmp8 = icmp ugt i64 %n, 1
- br i1 %cmp8, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:
- %idx0.lcssa = phi i64 [ 0, %entry ], [ %idx0.1, %for.body ]
- %idx1.lcssa = phi i64 [ 0, %entry ], [ %idx1.1, %for.body ]
- %val.lcssa = phi double [ %0, %entry ], [ %val.1, %for.body ]
- store double %val.lcssa, ptr %out.val, align 8
- store i64 %idx0.lcssa, ptr %out.idx0, align 8
- store i64 %idx1.lcssa, ptr %out.idx1, align 8
- ret void
-
-for.body:
- %src.phi = phi ptr [ %src.next, %for.body ], [ %src, %entry ]
- %iv = phi i64 [ %inc, %for.body ], [ 1, %entry ]
- %val.0 = phi double [ %val.1, %for.body ], [ %0, %entry ]
- %idx0.0 = phi i64 [ %idx0.1, %for.body ], [ 0, %entry ]
- %idx1.0 = phi i64 [ %idx1.1, %for.body ], [ 0, %entry ]
- %src.next = getelementptr inbounds nuw i8, ptr %src.phi, i64 8
- %1 = load double, ptr %src.next, align 8
- %cmp = fcmp fast ogt double %val.0, %1
- %idx0.1 = select i1 %cmp, i64 %iv, i64 %idx0.0
- %idx1.1 = select i1 %cmp, i64 %iv, i64 %idx1.0
- %val.1 = tail call nnan ninf nsz double @llvm.minnum.f64(double %val.0, double %1)
- %inc = add nuw i64 %iv, 1
- %exitcond = icmp eq i64 %inc, %n
- br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-; Unsupported: selected value is not the induction value.
-define i64 @smin_i32_idx_shifted_bad(ptr %src, i64 %n) {
-; CHECK-LABEL: define i64 @smin_i32_idx_shifted_bad(
-; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[IV_PLUS1:%.*]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_PLUS1]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN_VAL]], i32 [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
- %min.val = phi i32 [ 2147483647, %entry ], [ %min.val.next, %loop ]
- %gep = getelementptr i32, ptr %src, i64 %iv
- %l = load i32, ptr %gep, align 4
- %cmp = icmp sgt i32 %min.val, %l
- %iv.plus1 = add nuw nsw i64 %iv, 1
- %min.idx.next = select i1 %cmp, i64 %iv.plus1, i64 %min.idx
- %min.val.next = call i32 @llvm.smin.i32(i32 %min.val, i32 %l)
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %n
- br i1 %exitcond, label %exit, label %loop
-
-exit:
- %res = phi i64 [ %min.idx.next, %loop ]
- ret i64 %res
-}
>From 8c912b4956523aa8993b12874f683ace2d51aca3 Mon Sep 17 00:00:00 2001
From: nasmnc01 <nashe.mncube at arm.com>
Date: Mon, 8 Jun 2026 12:00:48 +0100
Subject: [PATCH 05/13] Reject FMaxNum/FMinNum patterns in
handleMultiUseReductions
FP maxnum/minnum have special semantics with respect to how they handle
signalling NaNs. As a consequence we shouldn't attempt to vectorize these
reduction patterns like we do with patterns using FP maximumnum/minimumnum.
Change-Id: I7717a7c9a1b10e0bd3b281b5bf78324d49d6ee6c
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index f1fe9b5c1dfd9..cde16c1724d93 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2180,14 +2180,16 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
case RecurKind::FMax:
case RecurKind::FMaximum:
- case RecurKind::FMaxNum:
case RecurKind::FMaximumNum:
return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT;
case RecurKind::FMin:
case RecurKind::FMinimum:
- case RecurKind::FMinNum:
case RecurKind::FMinimumNum:
return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT;
+ // minnum and maxnum need special handling due to expected sNaN behaviour
+ case RecurKind::FMinNum:
+ case RecurKind::FMaxNum:
+ return false;
default:
llvm_unreachable("unhandled recurrence kind");
}
>From ffc25e628aa0a3054ff81e43b105606c7f98ed30 Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Thu, 25 Jun 2026 14:41:29 +0100
Subject: [PATCH 06/13] Only allow un-ordered reductions for fmaximum/fminimum
Semantics of fmaximum/fminimum intrinsics requires reductions to only be
unordered to correctly handled NaN behaviour.
Change-Id: I924b5f47d4c92cf61b6bec31400dda8a603d54d7
---
.../Vectorize/VPlanConstruction.cpp | 6 +-
.../LoopVectorize/select-first-index-fp.ll | 102 +++---------------
.../LoopVectorize/select-last-index-fp.ll | 102 +++---------------
3 files changed, 36 insertions(+), 174 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index cde16c1724d93..48d57f46762d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2178,12 +2178,14 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SLE || Pred == CmpInst::ICMP_SLT;
case RecurKind::SMin:
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
- case RecurKind::FMax:
case RecurKind::FMaximum:
+ return Pred == CmpInst::FCMP_ULE || Pred == CmpInst::FCMP_ULT;
+ case RecurKind::FMax:
case RecurKind::FMaximumNum:
return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT;
- case RecurKind::FMin:
case RecurKind::FMinimum:
+ return Pred == CmpInst::FCMP_UGE || Pred == CmpInst::FCMP_UGT;
+ case RecurKind::FMin:
case RecurKind::FMinimumNum:
return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT;
// minnum and maxnum need special handling due to expected sNaN behaviour
diff --git a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
index 04916a11db6b0..126cbe57974ca 100644
--- a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
@@ -354,47 +354,12 @@ exit:
define i64 @test_fminimum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
@@ -402,9 +367,9 @@ define i64 @test_fminimum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -452,7 +417,7 @@ define i64 @test_fmaximumnum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
@@ -480,7 +445,7 @@ define i64 @test_fmaximumnum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -530,7 +495,7 @@ define i64 @test_fminimumnum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
@@ -558,7 +523,7 @@ define i64 @test_fminimumnum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -588,47 +553,12 @@ exit:
define i64 @test_fmaximum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
@@ -636,9 +566,9 @@ define i64 @test_fmaximum_first_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index 389ace0c29570..fd432a3a99c13 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -263,47 +263,12 @@ exit:
define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
@@ -311,9 +276,9 @@ define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -361,7 +326,7 @@ define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
@@ -389,7 +354,7 @@ define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
@@ -419,47 +384,12 @@ exit:
define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
@@ -467,9 +397,9 @@ define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -517,7 +447,7 @@ define i64 @test_fmaximumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
@@ -545,7 +475,7 @@ define i64 @test_fmaximumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
>From 7e608092c87e18a871c62975c8ca3044f12d43ec Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Wed, 1 Jul 2026 10:06:42 +0100
Subject: [PATCH 07/13] Unordered EQ for fmaximum/fminimum reductions
Change-Id: I9403664cf7656df55d5157eaba05f42ce0128c5a
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 48d57f46762d9..d305d0936211b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2257,7 +2257,11 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
auto *FinalMinOrMaxCmp =
(RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind))
? B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult)
- : B.createFCmp(CmpInst::FCMP_OEQ, MinOrMaxExiting, MinOrMaxResult);
+ : B.createFCmp((RdxKind == RecurKind::FMaximum ||
+ RdxKind == RecurKind::FMinimum)
+ ? CmpInst::FCMP_UEQ
+ : CmpInst::FCMP_OEQ,
+ MinOrMaxExiting, MinOrMaxResult);
VPValue *Sentinel = FindIVCmp->getOperand(1);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
>From 096730e372298624b21fdc4e78989d501530b46e Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Thu, 9 Jul 2026 15:16:14 +0100
Subject: [PATCH 08/13] Add argmin/argmax tests for fp
Change-Id: Id5d3a7220bc16d19720b713cdb08a870c4d51661
---
.../LoopVectorize/multiple-argmin-argmax.ll | 251 ++++++++++++++++++
1 file changed, 251 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll b/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
index 8fc5e4a17122c..46f0c53344569 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
@@ -573,4 +573,255 @@ exit:
ret i64 %res
}
+define i64 @float_argminimum_argmaximum(ptr %data, float %start_val) {
+; CHECK-LABEL: define i64 @float_argminimum_argmaximum(
+; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[START_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp ult <2 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP4]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp ugt <2 x float> [[WIDE_LOAD]], [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP5]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP7]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> [[TMP4]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp ueq <2 x float> [[TMP4]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP3]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], -9223372036854775808
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> [[TMP7]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT6]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp ueq <2 x float> [[TMP7]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP16]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP17]])
+; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP18]], -9223372036854775808
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 99, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[TMP9]], %[[SCALAR_PH]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ [[TMP14]], %[[SCALAR_PH]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[TMP15]], %[[SCALAR_PH]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp ult float [[VAL]], [[MINVAL]]
+; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
+; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minimum.f32(float [[VAL]], float [[MINVAL]])
+; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ugt float [[VAL]], [[MAXVAL]]
+; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
+; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maximum.f32(float [[VAL]], float [[MAXVAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
+ %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
+ %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
+ %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
+ %gep = getelementptr inbounds float, ptr %data, i64 %iv
+ %val = load float, ptr %gep, align 4
+ %cmp_min = fcmp ult float %val, %minval
+ %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
+ %new_minval = call float @llvm.minimum.f32(float %val, float %minval)
+ %cmp_max = fcmp ugt float %val, %maxval
+ %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
+ %new_maxval = call float @llvm.maximum.f32(float %val, float %maxval)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit_cond = icmp eq i64 %iv.next, 100
+ br i1 %exit_cond, label %exit, label %loop
+
+exit:
+ %res = add i64 %new_minpos, %new_maxpos
+ ret i64 %res
+}
+
+
+define i64 @float_argminimumnum_argmaximumnum(ptr %data, float %start_val) {
+; CHECK-LABEL: define i64 @float_argminimumnum_argmaximumnum(
+; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[START_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp olt <2 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP4]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp ogt <2 x float> [[WIDE_LOAD]], [[VEC_PHI2]]
+; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP5]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI3]]
+; CHECK-NEXT: [[TMP7]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmin.v2f32(<2 x float> [[TMP4]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = fcmp oeq <2 x float> [[TMP4]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP3]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], -9223372036854775808
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = call float @llvm.vector.reduce.fmax.v2f32(<2 x float> [[TMP7]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT6]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = fcmp oeq <2 x float> [[TMP7]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP16]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP17]])
+; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP18]], -9223372036854775808
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 99, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[TMP9]], %[[SCALAR_PH]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ [[TMP14]], %[[SCALAR_PH]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[TMP15]], %[[SCALAR_PH]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp olt float [[VAL]], [[MINVAL]]
+; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
+; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minimumnum.f32(float [[VAL]], float [[MINVAL]])
+; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ogt float [[VAL]], [[MAXVAL]]
+; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
+; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maximumnum.f32(float [[VAL]], float [[MAXVAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
+ %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
+ %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
+ %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
+ %gep = getelementptr inbounds float, ptr %data, i64 %iv
+ %val = load float, ptr %gep, align 4
+ %cmp_min = fcmp olt float %val, %minval
+ %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
+ %new_minval = call float @llvm.minimumnum.f32(float %val, float %minval)
+ %cmp_max = fcmp ogt float %val, %maxval
+ %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
+ %new_maxval = call float @llvm.maximumnum.f32(float %val, float %maxval)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit_cond = icmp eq i64 %iv.next, 100
+ br i1 %exit_cond, label %exit, label %loop
+
+exit:
+ %res = add i64 %new_minpos, %new_maxpos
+ ret i64 %res
+}
+
+define i64 @float_argminnum_argmaxnum(ptr %data, float %start_val) {
+; CHECK-LABEL: define i64 @float_argmininum_argmaxinum(
+; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[START_VAL]], %[[ENTRY]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[START_VAL]], %[[ENTRY]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp olt float [[VAL]], [[MINVAL]]
+; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
+; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minnum.f32(float [[VAL]], float [[MINVAL]])
+; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ogt float [[VAL]], [[MAXVAL]]
+; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
+; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maxnum.f32(float [[VAL]], float [[MAXVAL]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
+ %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
+ %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
+ %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
+ %gep = getelementptr inbounds float, ptr %data, i64 %iv
+ %val = load float, ptr %gep, align 4
+ %cmp_min = fcmp olt float %val, %minval
+ %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
+ %new_minval = call float @llvm.minnum.f32(float %val, float %minval)
+ %cmp_max = fcmp ogt float %val, %maxval
+ %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
+ %new_maxval = call float @llvm.maxnum.f32(float %val, float %maxval)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit_cond = icmp eq i64 %iv.next, 100
+ br i1 %exit_cond, label %exit, label %loop
+
+exit:
+ %res = add i64 %new_minpos, %new_maxpos
+ ret i64 %res
+}
+
>From 99fc808a69ebc4a84065cb804b40462f397cbeff Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Tue, 21 Jul 2026 13:26:56 +0100
Subject: [PATCH 09/13] Strict predicate check on FP
Change-Id: If5833b47896b7deb09dc20e067f4866e7bbb71b2
---
.../Vectorize/VPlanConstruction.cpp | 7 +-
.../LoopVectorize/multiple-argmin-argmax.ll | 251 ------------------
.../LoopVectorize/select-first-index-fp.ll | 150 +++--------
.../LoopVectorize/select-last-index-fp.ll | 48 ++--
4 files changed, 70 insertions(+), 386 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index d305d0936211b..2e9201d31eb52 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1938,6 +1938,11 @@ static bool handleFirstArgMinOrMax(
assert(FindLastIVPhiR->getVFScaleFactor() == 1 &&
"FindIV reduction must not be scaled");
+ // TODO: support for FP in handleFirstArgMinOrMax
+ if (RecurrenceDescriptor::isFloatingPointRecurrenceKind(
+ MinOrMaxPhiR->getRecurrenceKind()))
+ return false;
+
Type *Ty = Plan.getVectorLoopRegion()->getCanonicalIVType();
// TODO: Support non (i.e., narrower than) canonical IV types.
// TODO: Emit remarks for failed transformations.
@@ -2219,7 +2224,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
MinOrMaxResult->moveBefore(*FindIVRdxResult->getParent(),
FindIVRdxResult->getIterator());
- bool IsStrictPredicate = ICmpInst::isLT(Pred) || ICmpInst::isGT(Pred);
+ bool IsStrictPredicate = CmpInst::isStrictPredicate(Pred);
if (IsStrictPredicate) {
if (!handleFirstArgMinOrMax(Plan, MinOrMaxPhiR, FindIVPhiR,
cast<VPWidenIntOrFpInductionRecipe>(IVOp),
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll b/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
index 46f0c53344569..8fc5e4a17122c 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-argmin-argmax.ll
@@ -573,255 +573,4 @@ exit:
ret i64 %res
}
-define i64 @float_argminimum_argmaximum(ptr %data, float %start_val) {
-; CHECK-LABEL: define i64 @float_argminimum_argmaximum(
-; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[START_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp ult <2 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP4]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI]])
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp ugt <2 x float> [[WIDE_LOAD]], [[VEC_PHI2]]
-; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP5]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI3]]
-; CHECK-NEXT: [[TMP7]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> [[TMP4]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp ueq <2 x float> [[TMP4]], [[BROADCAST_SPLAT5]]
-; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP3]], <2 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], -9223372036854775808
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> [[TMP7]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT6]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp ueq <2 x float> [[TMP7]], [[BROADCAST_SPLAT7]]
-; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP16]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP17]])
-; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP18]], -9223372036854775808
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 0
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 99, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[TMP9]], %[[SCALAR_PH]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ [[TMP14]], %[[SCALAR_PH]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[TMP15]], %[[SCALAR_PH]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp ult float [[VAL]], [[MINVAL]]
-; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
-; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minimum.f32(float [[VAL]], float [[MINVAL]])
-; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ugt float [[VAL]], [[MAXVAL]]
-; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
-; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maximum.f32(float [[VAL]], float [[MAXVAL]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
- %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
- %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
- %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
- %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
- %gep = getelementptr inbounds float, ptr %data, i64 %iv
- %val = load float, ptr %gep, align 4
- %cmp_min = fcmp ult float %val, %minval
- %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
- %new_minval = call float @llvm.minimum.f32(float %val, float %minval)
- %cmp_max = fcmp ugt float %val, %maxval
- %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
- %new_maxval = call float @llvm.maximum.f32(float %val, float %maxval)
- %iv.next = add nuw nsw i64 %iv, 1
- %exit_cond = icmp eq i64 %iv.next, 100
- br i1 %exit_cond, label %exit, label %loop
-
-exit:
- %res = add i64 %new_minpos, %new_maxpos
- ret i64 %res
-}
-
-
-define i64 @float_argminimumnum_argmaximumnum(ptr %data, float %start_val) {
-; CHECK-LABEL: define i64 @float_argminimumnum_argmaximumnum(
-; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[START_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp olt <2 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI1]]
-; CHECK-NEXT: [[TMP4]] = call <2 x float> @llvm.minimumnum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI]])
-; CHECK-NEXT: [[TMP5:%.*]] = fcmp ogt <2 x float> [[WIDE_LOAD]], [[VEC_PHI2]]
-; CHECK-NEXT: [[TMP6]] = select <2 x i1> [[TMP5]], <2 x i64> [[VEC_IND]], <2 x i64> [[VEC_PHI3]]
-; CHECK-NEXT: [[TMP7]] = call <2 x float> @llvm.maximumnum.v2f32(<2 x float> [[WIDE_LOAD]], <2 x float> [[VEC_PHI2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 98
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fmin.v2f32(<2 x float> [[TMP4]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x float> poison, float [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT4]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp oeq <2 x float> [[TMP4]], [[BROADCAST_SPLAT5]]
-; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP10]], <2 x i64> [[TMP3]], <2 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i64 [[TMP12]], -9223372036854775808
-; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = call float @llvm.vector.reduce.fmax.v2f32(<2 x float> [[TMP7]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT6]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = fcmp oeq <2 x float> [[TMP7]], [[BROADCAST_SPLAT7]]
-; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP16]], <2 x i64> [[TMP6]], <2 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[TMP17]])
-; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i64 [[TMP18]], -9223372036854775808
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP18]], i64 0
-; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 99, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[TMP9]], %[[SCALAR_PH]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ [[TMP14]], %[[SCALAR_PH]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[TMP15]], %[[SCALAR_PH]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ [[TMP20]], %[[SCALAR_PH]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp olt float [[VAL]], [[MINVAL]]
-; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
-; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minimumnum.f32(float [[VAL]], float [[MINVAL]])
-; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ogt float [[VAL]], [[MAXVAL]]
-; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
-; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maximumnum.f32(float [[VAL]], float [[MAXVAL]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
- %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
- %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
- %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
- %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
- %gep = getelementptr inbounds float, ptr %data, i64 %iv
- %val = load float, ptr %gep, align 4
- %cmp_min = fcmp olt float %val, %minval
- %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
- %new_minval = call float @llvm.minimumnum.f32(float %val, float %minval)
- %cmp_max = fcmp ogt float %val, %maxval
- %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
- %new_maxval = call float @llvm.maximumnum.f32(float %val, float %maxval)
- %iv.next = add nuw nsw i64 %iv, 1
- %exit_cond = icmp eq i64 %iv.next, 100
- br i1 %exit_cond, label %exit, label %loop
-
-exit:
- %res = add i64 %new_minpos, %new_maxpos
- ret i64 %res
-}
-
-define i64 @float_argminnum_argmaxnum(ptr %data, float %start_val) {
-; CHECK-LABEL: define i64 @float_argmininum_argmaxinum(
-; CHECK-SAME: ptr [[DATA:%.*]], float [[START_VAL:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINVAL:%.*]] = phi float [ [[START_VAL]], %[[ENTRY]] ], [ [[NEW_MINVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MINPOS:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEW_MINPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXVAL:%.*]] = phi float [ [[START_VAL]], %[[ENTRY]] ], [ [[NEW_MAXVAL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAXPOS:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEW_MAXPOS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP_MIN:%.*]] = fcmp olt float [[VAL]], [[MINVAL]]
-; CHECK-NEXT: [[NEW_MINPOS]] = select i1 [[CMP_MIN]], i64 [[IV]], i64 [[MINPOS]]
-; CHECK-NEXT: [[NEW_MINVAL]] = call float @llvm.minnum.f32(float [[VAL]], float [[MINVAL]])
-; CHECK-NEXT: [[CMP_MAX:%.*]] = fcmp ogt float [[VAL]], [[MAXVAL]]
-; CHECK-NEXT: [[NEW_MAXPOS]] = select i1 [[CMP_MAX]], i64 [[IV]], i64 [[MAXPOS]]
-; CHECK-NEXT: [[NEW_MAXVAL]] = call float @llvm.maxnum.f32(float [[VAL]], float [[MAXVAL]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[NEW_MINPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MINPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[NEW_MAXPOS_LCSSA:%.*]] = phi i64 [ [[NEW_MAXPOS]], %[[LOOP]] ]
-; CHECK-NEXT: [[RES:%.*]] = add i64 [[NEW_MINPOS_LCSSA]], [[NEW_MAXPOS_LCSSA]]
-; CHECK-NEXT: ret i64 [[RES]]
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
- %minval = phi float [ %start_val, %entry ], [ %new_minval, %loop ]
- %minpos = phi i64 [ 0, %entry ], [ %new_minpos, %loop ]
- %maxval = phi float [ %start_val, %entry ], [ %new_maxval, %loop ]
- %maxpos = phi i64 [ 0, %entry ], [ %new_maxpos, %loop ]
- %gep = getelementptr inbounds float, ptr %data, i64 %iv
- %val = load float, ptr %gep, align 4
- %cmp_min = fcmp olt float %val, %minval
- %new_minpos = select i1 %cmp_min, i64 %iv, i64 %minpos
- %new_minval = call float @llvm.minnum.f32(float %val, float %minval)
- %cmp_max = fcmp ogt float %val, %maxval
- %new_maxpos = select i1 %cmp_max, i64 %iv, i64 %maxpos
- %new_maxval = call float @llvm.maxnum.f32(float %val, float %maxval)
- %iv.next = add nuw nsw i64 %iv, 1
- %exit_cond = icmp eq i64 %iv.next, 100
- br i1 %exit_cond, label %exit, label %loop
-
-exit:
- %res = add i64 %new_minpos, %new_maxpos
- ret i64 %res
-}
-
diff --git a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
index 126cbe57974ca..4ab5eaee6b749 100644
--- a/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-first-index-fp.ll
@@ -354,22 +354,22 @@ exit:
define i64 @test_fminimum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -398,56 +398,21 @@ define i64 @test_fmaximumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan olt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -476,56 +441,21 @@ define i64 @test_fminimumnum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp nnan ogt <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2]] = call nnan <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
-; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP2]])
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -553,22 +483,22 @@ exit:
define i64 @test_fmaximum_first_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_first_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index fd432a3a99c13..99726fce0d8d9 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -263,22 +263,22 @@ exit:
define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
-; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -384,22 +384,22 @@ exit:
define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[LOOP1:.*]]
-; CHECK: [[LOOP1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimum.f32(float [[L]], float [[MIN_VAL]])
-; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MIN_IDX]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
>From 1b975adefac48508100c198ae210d8bb32e574cf Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Mon, 10 Aug 2026 14:48:54 +0100
Subject: [PATCH 10/13] fmaximum/fminimum review comments
Change-Id: I8a7983562b03d60b08e5fd4c6c5a08e766f276a4
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 2e9201d31eb52..a075c9b40c8b5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2183,19 +2183,18 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SLE || Pred == CmpInst::ICMP_SLT;
case RecurKind::SMin:
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
- case RecurKind::FMaximum:
- return Pred == CmpInst::FCMP_ULE || Pred == CmpInst::FCMP_ULT;
case RecurKind::FMax:
case RecurKind::FMaximumNum:
return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT;
- case RecurKind::FMinimum:
- return Pred == CmpInst::FCMP_UGE || Pred == CmpInst::FCMP_UGT;
case RecurKind::FMin:
case RecurKind::FMinimumNum:
return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT;
// minnum and maxnum need special handling due to expected sNaN behaviour
+ // minimum and maximum return NaN if either input is a NAN
case RecurKind::FMinNum:
case RecurKind::FMaxNum:
+ case RecurKind::FMinimum:
+ case RecurKind::FMaximum:
return false;
default:
llvm_unreachable("unhandled recurrence kind");
>From 6b6f7750996248eec38f87225479d68a153cefe4 Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Tue, 11 Aug 2026 10:31:44 +0100
Subject: [PATCH 11/13] Rebase and update tests
Change-Id: I8550cc92a593c92e9e9977b0031b54f55ed602af
---
llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index 99726fce0d8d9..59ce2bc1f61c9 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -310,7 +310,7 @@ define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -431,7 +431,7 @@ define i64 @test_fmaximumnum_last_idx(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
>From 096dce6ce3ac677f6e9d51201fe40bc229ac2b55 Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Tue, 18 Aug 2026 16:16:54 +0100
Subject: [PATCH 12/13] Add strict/non-strict predicate tests
- add testing for strict/non-strict predicates in reductions
- tests for NaN difference between semantic fmax variants
Change-Id: I25d3da70317bfbef525076ef37c6fbc99ec93d62
---
.../LoopVectorize/select-last-index-fp.ll | 430 ++++++++++++++++++
1 file changed, 430 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index 59ce2bc1f61c9..c14a8010e1442 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -87,6 +87,50 @@ exit:
ret i64 %res
}
+; fmax last-index with strict predicate should not vectorize: FP strict
+; predicates still cannot use the first-argmax transform.
+define i64 @test_fmax_last_idx_strict(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmax_last_idx_strict(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan olt float %max.val, %l
+ %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmin last-index with non-strict predicate (oge: min.val >= l).
define i64 @test_fmin_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmin_last_idx(
@@ -173,6 +217,50 @@ exit:
ret i64 %res
}
+; fmin last-index with strict predicate should not vectorize: FP strict
+; predicates still cannot use the first-argmin transform.
+define i64 @test_fmin_last_idx_strict(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmin_last_idx_strict(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan ogt float %min.val, %l
+ %min.val.next = call nnan float @llvm.minnum.f32(float %l, float %min.val)
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmax last-index without nnan: should NOT vectorize via the argmax path.
define i64 @test_fmax_last_idx_no_nnan(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmax_last_idx_no_nnan(
@@ -259,6 +347,94 @@ exit:
ret i64 %res
}
+; fmax last-index using llvm.maxnum intrinsic (FMaxNum recurrence kind).
+define i64 @test_fmaxnum_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaxnum_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan ole float %max.val, %l
+ %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+; fmax last-index using llvm.maxnum without nnan remains scalar. Unlike
+; llvm.maximumnum, last-index vectorization does not support maxnum NaN
+; semantics here.
+define i64 @test_fmaxnum_last_idx_no_nnan(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaxnum_last_idx_no_nnan(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp ole float %max.val, %l
+ %max.val.next = call float @llvm.maxnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmax last-index using llvm.maximum intrinsic (FMaximum recurrence kind).
define i64 @test_fmaximum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmaximum_last_idx(
@@ -302,6 +478,93 @@ exit:
ret i64 %res
}
+; fmax last-index using llvm.maximum with an unordered compare remains scalar.
+; maximum propagates NaNs, so this shape cannot use the last-index transform.
+define i64 @test_fmaximum_last_idx_unordered(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaximum_last_idx_unordered(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ule float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call float @llvm.maximum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp ule float %max.val, %l
+ %max.val.next = call float @llvm.maximum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+; fmin last-index using llvm.minnum intrinsic (FMinNum recurrence kind).
+define i64 @test_fminnum_last_idx(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fminnum_last_idx(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan oge float %min.val, %l
+ %min.val.next = call nnan float @llvm.minnum.f32(float %l, float %min.val)
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmin last-index using llvm.minimumnum intrinsic (FMinimumNum recurrence kind).
define i64 @test_fminimumnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimumnum_last_idx(
@@ -380,6 +643,50 @@ exit:
ret i64 %res
}
+; fmin last-index using llvm.minimumnum with a strict predicate should not
+; vectorize.
+define i64 @test_fminimumnum_last_idx_strict(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fminimumnum_last_idx_strict(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan ogt float %min.val, %l
+ %min.val.next = call nnan float @llvm.minimumnum.f32(float %l, float %min.val)
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmin last-index using llvm.minimum intrinsic (FMinimum recurrence kind).
define i64 @test_fminimum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fminimum_last_idx(
@@ -501,6 +808,129 @@ exit:
ret i64 %res
}
+; fmax last-index using llvm.maximumnum without nnan still vectorizes. This
+; distinguishes maximumnum from maxnum for NaN-sensitive inputs.
+define i64 @test_fmaximumnum_last_idx_no_nnan(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaximumnum_last_idx_no_nnan(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2]] = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP3]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP8]], -9223372036854775808
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp ole float %max.val, %l
+ %max.val.next = call float @llvm.maximumnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+; fmax last-index using llvm.maximumnum with a strict predicate should not
+; vectorize.
+define i64 @test_fmaximumnum_last_idx_strict(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaximumnum_last_idx_strict(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan olt float %max.val, %l
+ %max.val.next = call nnan float @llvm.maximumnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
define i64 @test_select_fmaxnum_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_select_fmaxnum_last_idx(
>From 4dfa862d052b37bd5079b050fe0473c7949dcffd Mon Sep 17 00:00:00 2001
From: Nashe Mncube <nashe.mncube at arm.com>
Date: Wed, 26 Aug 2026 12:48:04 +0100
Subject: [PATCH 13/13] Review comments
Change-Id: I96ff5767abb4bb3c920ad831401f6f1da4a38b2f
---
.../Vectorize/VPlanConstruction.cpp | 16 +-
.../LoopVectorize/select-last-index-fp.ll | 387 +++++++++++++++---
2 files changed, 342 insertions(+), 61 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a075c9b40c8b5..0c240b79ca757 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -2212,6 +2212,16 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return false;
}
+ if (RdxKind == RecurKind::FMaximumNum ||
+ RdxKind == RecurKind::FMinimumNum) {
+ auto *StartValue = dyn_cast<VPIRValue>(MinOrMaxPhiR->getStartValue());
+ auto *StartC = StartValue
+ ? dyn_cast<ConstantFP>(StartValue->getLiveInIRValue())
+ : nullptr;
+ if (!StartC || StartC->isNaN())
+ return false;
+ }
+
auto *FindIVSelect = findFindIVSelect(FindIVPhiR->getBackedgeValue());
auto *FindIVCmp = FindIVSelect->getOperand(0)->getDefiningRecipe();
auto *FindIVRdxResult = cast<VPInstruction>(FindIVCmp->getOperand(0));
@@ -2261,11 +2271,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
auto *FinalMinOrMaxCmp =
(RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind))
? B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult)
- : B.createFCmp((RdxKind == RecurKind::FMaximum ||
- RdxKind == RecurKind::FMinimum)
- ? CmpInst::FCMP_UEQ
- : CmpInst::FCMP_OEQ,
- MinOrMaxExiting, MinOrMaxResult);
+ : B.createFCmp(CmpInst::FCMP_OEQ, MinOrMaxExiting, MinOrMaxResult);
VPValue *Sentinel = FindIVCmp->getOperand(1);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
diff --git a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
index c14a8010e1442..9b5f164fbde11 100644
--- a/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-last-index-fp.ll
@@ -6,21 +6,56 @@ define i64 @test_fmax_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmax_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp nnan nsz ole <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call nnan nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp oeq <4 x float> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan nsz float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP1_META:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -32,8 +67,8 @@ loop:
%max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan ole float %max.val, %l
- %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
+ %cmp = fcmp nnan nsz ole float %max.val, %l
+ %max.val.next = call nnan nsz float @llvm.maxnum.f32(float %l, float %max.val)
%max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -49,21 +84,56 @@ define i64 @test_fmax_last_idx_cond_flipped(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmax_last_idx_cond_flipped(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0xFF7FFFFF), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp nnan nsz oge <4 x float> [[WIDE_LOAD]], [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP3]] = call nnan nsz <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp oeq <4 x float> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ f0xFF7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[L]], [[MAX_VAL]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz oge float [[L]], [[MAX_VAL]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan nsz float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -75,8 +145,8 @@ loop:
%max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan oge float %l, %max.val
- %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
+ %cmp = fcmp nnan nsz oge float %l, %max.val
+ %max.val.next = call nnan nsz float @llvm.maxnum.f32(float %l, float %max.val)
%max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -100,8 +170,8 @@ define i64 @test_fmax_last_idx_strict(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan olt float [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz olt float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan nsz float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
@@ -119,8 +189,8 @@ loop:
%max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan olt float %max.val, %l
- %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
+ %cmp = fcmp nnan nsz olt float %max.val, %l
+ %max.val.next = call nnan nsz float @llvm.maxnum.f32(float %l, float %max.val)
%max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -136,21 +206,56 @@ define i64 @test_fmin_last_idx(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmin_last_idx(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp nnan nsz oge <4 x float> [[VEC_PHI1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3]] = call nnan nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp oeq <4 x float> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan oge float [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz oge float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan nsz float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -162,8 +267,8 @@ loop:
%min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan oge float %min.val, %l
- %min.val.next = call nnan float @llvm.minnum.f32(float %l, float %min.val)
+ %cmp = fcmp nnan nsz oge float %min.val, %l
+ %min.val.next = call nnan nsz float @llvm.minnum.f32(float %l, float %min.val)
%min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -179,21 +284,56 @@ define i64 @test_fmin_last_idx_cond_flipped(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmin_last_idx_cond_flipped(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float f0x7F7FFFFF), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp nnan nsz ole <4 x float> [[WIDE_LOAD]], [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP3]] = call nnan nsz <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI1]])
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp oeq <4 x float> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ f0x7F7FFFFF, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[L]], [[MIN_VAL]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz ole float [[L]], [[MIN_VAL]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan nsz float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -205,8 +345,8 @@ loop:
%min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan ole float %l, %min.val
- %min.val.next = call nnan float @llvm.minnum.f32(float %l, float %min.val)
+ %cmp = fcmp nnan nsz ole float %l, %min.val
+ %min.val.next = call nnan nsz float @llvm.minnum.f32(float %l, float %min.val)
%min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -230,8 +370,8 @@ define i64 @test_fmin_last_idx_strict(ptr %src, i64 %n) {
; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ogt float [[MIN_VAL]], [[L]]
-; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan nsz ogt float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call nnan nsz float @llvm.minnum.f32(float [[L]], float [[MIN_VAL]])
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
@@ -249,8 +389,8 @@ loop:
%min.val = phi float [ 0x47EFFFFFE0000000, %entry ], [ %min.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp nnan ogt float %min.val, %l
- %min.val.next = call nnan float @llvm.minnum.f32(float %l, float %min.val)
+ %cmp = fcmp nnan nsz ogt float %min.val, %l
+ %min.val.next = call nnan nsz float @llvm.minnum.f32(float %l, float %min.val)
%min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -262,6 +402,7 @@ exit:
}
; fmax last-index without nnan: should NOT vectorize via the argmax path.
+; This keeps nsz so missing nnan is the only blocker.
define i64 @test_fmax_last_idx_no_nnan(ptr %src, i64 %n) {
; CHECK-LABEL: define i64 @test_fmax_last_idx_no_nnan(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
@@ -273,8 +414,8 @@ define i64 @test_fmax_last_idx_no_nnan(ptr %src, i64 %n) {
; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ole float [[MAX_VAL]], [[L]]
-; CHECK-NEXT: [[MAX_VAL_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nsz ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nsz float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
@@ -292,8 +433,52 @@ loop:
%max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
%gep = getelementptr float, ptr %src, i64 %iv
%l = load float, ptr %gep
- %cmp = fcmp ole float %max.val, %l
- %max.val.next = call float @llvm.maxnum.f32(float %l, float %max.val)
+ %cmp = fcmp nsz ole float %max.val, %l
+ %max.val.next = call nsz float @llvm.maxnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
+; fmax last-index without nsz: should NOT vectorize via the argmax path.
+; This keeps nnan so missing nsz is the only blocker.
+define i64 @test_fmax_last_idx_no_nsz(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmax_last_idx_no_nsz(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call nnan float @llvm.maxnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0xC7EFFFFFE0000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp nnan ole float %max.val, %l
+ %max.val.next = call nnan float @llvm.maxnum.f32(float %l, float %max.val)
%max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
@@ -643,6 +828,51 @@ exit:
ret i64 %res
}
+; fmin last-index using llvm.minimumnum with a NaN start value should remain
+; scalar because the multi-use last-index transform cannot prove the seed is
+; non-NaN.
+define i64 @test_fminimumnum_last_idx_nan_start(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fminimumnum_last_idx_nan_start(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_VAL:%.*]] = phi float [ +qnan, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp oge float [[MIN_VAL]], [[L]]
+; CHECK-NEXT: [[MIN_VAL_NEXT]] = call float @llvm.minimumnum.f32(float [[L]], float [[MIN_VAL]])
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.val = phi float [ 0x7FF8000000000000, %entry ], [ %min.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp oge float %min.val, %l
+ %min.val.next = call float @llvm.minimumnum.f32(float %l, float %min.val)
+ %min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %min.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmin last-index using llvm.minimumnum with a strict predicate should not
; vectorize.
define i64 @test_fminimumnum_last_idx_strict(ptr %src, i64 %n) {
@@ -808,6 +1038,51 @@ exit:
ret i64 %res
}
+; fmax last-index using llvm.maximumnum with a NaN start value should remain
+; scalar because the multi-use last-index transform cannot prove the seed is
+; non-NaN.
+define i64 @test_fmaximumnum_last_idx_nan_start(ptr %src, i64 %n) {
+; CHECK-LABEL: define i64 @test_fmaximumnum_last_idx_nan_start(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_VAL:%.*]] = phi float [ +qnan, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ole float [[MAX_VAL]], [[L]]
+; CHECK-NEXT: [[MAX_VAL_NEXT]] = call float @llvm.maximumnum.f32(float [[L]], float [[MAX_VAL]])
+; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
+ %max.val = phi float [ 0x7FF8000000000000, %entry ], [ %max.val.next, %loop ]
+ %gep = getelementptr float, ptr %src, i64 %iv
+ %l = load float, ptr %gep
+ %cmp = fcmp ole float %max.val, %l
+ %max.val.next = call float @llvm.maximumnum.f32(float %l, float %max.val)
+ %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ %res = phi i64 [ %max.idx.next, %loop ]
+ ret i64 %res
+}
+
; fmax last-index using llvm.maximumnum without nnan still vectorizes. This
; distinguishes maximumnum from maxnum for NaN-sensitive inputs.
define i64 @test_fmaximumnum_last_idx_no_nnan(ptr %src, i64 %n) {
More information about the llvm-commits
mailing list