[llvm] [LoopVectorize] Recognize conditional FP reductions without fast-math (PR #219414)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 01:45:11 PDT 2026
https://github.com/mkovacevic99 created https://github.com/llvm/llvm-project/pull/219414
`if (c[i]) s += a[i]` (branch form) and `if (a[i] > 0) s += a[i]` (select form) failed to vectorize without -ffast-math, even though masked ordered reduction instructions (RVV vfredosum.vs, SVE FADDA) sum active lanes in strict order and are bit-identical to the scalar loop. Recognition just never looked through the extra blend the condition introduces.
- checkOrderedReduction: look through the blend (phi or select) to find the real fadd/fmuladd underneath.
- isConditionalRdxPattern: allow FAdd/FSub selects without fast-math, tracking the exact-FP instruction instead.
- getReductionOpChain: same blend look-through, needed to keep the reduction in-loop (excluded for min/max, whose select already is the reduction op).
- createInLoopReductionRecipes: fold away the conditional-select the same way an existing phi blend is folded, but also erase it -- a select, unlike a phi, can't be left pointing at a value defined later in the block.
Update if-reduction.ll for the two fsub-select cases that now vectorize instead of failing at recognition.
Updated tail-folding-ordered-reduction.ll test with the branch and select form.
I discovered the missed optimization on RISCV but it turned out that there was already an open issue for ARM, so this PR also solves it for ARM as well as for RISCV https://github.com/llvm/llvm-project/issues/196703
<html>
<body>
<!--StartFragment--><p data-sourcepos="1:1-1:67" dir="auto">The performance measurements (measured on SpacemiT K3, X100 cores):</p>
kernel | n=64 | n=256 | n=1024 | n=4096
-- | -- | -- | -- | --
f32_cond_sum | 1.30× | 1.57× | 2.70× | 2.87×
f32_cond_sum_select | 0.95× | 1.83× | 3.21× | 3.42×
f32_plain_sum (control) | 1.00× | 1.00× | 1.00× | 1.00×
f64_cond_sum | 0.94× | 1.25× | 2.65× | 2.87×
f64_cond_sum_select | 1.29× | 1.96× | 3.29× | 3.46×
f64_plain_sum (control) | 1.00× | 1.00× | 1.00× | 1.00×
<!--EndFragment-->
</body>
</html>
Test I used:
```
float cond_sum(const float *restrict a, const int *restrict c, int n) {
float s = 0.f;
for (int i = 0; i < n; i++)
if (c[i]) s += a[i];
return s;
}
float cond_sum_select(const float *restrict a, int n) {
float s = 0.f;
for (int i = 0; i < n; i++)
if (a[i] > 0.f) s += a[i];
return s;
}
float plain_sum(const float *restrict a, int n) {
float s = 0.f;
for (int i = 0; i < n; i++) s += a[i];
return s;
}
```
>From da487c494d238ce953b60362c1a393bb49a0992b Mon Sep 17 00:00:00 2001
From: Milica Kovacevic <mkovacevic at baylibre.com>
Date: Thu, 27 Aug 2026 15:04:48 +0200
Subject: [PATCH] [LoopVectorize] Recognize conditional FP reductions without
fast-math
`if (c[i]) s += a[i]` (branch form) and `if (a[i] > 0) s += a[i]`
(select form) failed to vectorize without -ffast-math, even though
masked ordered reduction instructions (RVV vfredosum.vs, SVE FADDA)
sum active lanes in strict order and are bit-identical to the scalar
loop. Recognition just never looked through the extra blend the
condition introduces.
- checkOrderedReduction: look through the blend (phi or select) to
find the real fadd/fmuladd underneath.
- isConditionalRdxPattern: allow FAdd/FSub selects without fast-math,
tracking the exact-FP instruction instead.
- getReductionOpChain: same blend look-through, needed to keep the
reduction in-loop (excluded for min/max, whose select already is
the reduction op).
- createInLoopReductionRecipes: fold away the conditional-select the
same way an existing phi blend is folded, but also erase it -- a
select, unlike a phi, can't be left pointing at a value defined
later in the block.
Update if-reduction.ll for the two fsub-select cases that now
vectorize instead of failing at recognition.
Updated tail-folding-ordered-reduction.ll test with the branch
and select form.
---
llvm/lib/Analysis/IVDescriptors.cpp | 122 +++++++++--
.../Vectorize/VPlanConstruction.cpp | 43 ++++
.../RISCV/tail-folding-ordered-reduction.ll | 196 ++++++++++++++++++
.../Transforms/LoopVectorize/if-reduction.ll | 108 +++++++---
4 files changed, 420 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 800b64e9a29af..9ce5750e8b221 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -192,6 +192,51 @@ static bool checkOrderedReduction(RecurKind Kind, Instruction *ExactFPMathInst,
if (Kind != RecurKind::FAdd && Kind != RecurKind::FMulAdd)
return false;
+ // A conditional reduction merges the updated and unchanged values through
+ // a phi (branch form) or a select (speculated form), instead of exiting
+ // directly on the fadd/fmuladd. Look through either to find the real exit
+ // instruction.
+ if (isa<PHINode>(Exit) || isa<SelectInst>(Exit)) {
+ Instruction *Chain = nullptr;
+ if (auto *ExitPhi = dyn_cast<PHINode>(Exit)) {
+ if (ExitPhi->getNumIncomingValues() != 2)
+ return false;
+
+ Instruction *Inc0 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(0));
+ Instruction *Inc1 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(1));
+
+ // One incoming value is the reduction phi itself (carried forward when
+ // the update did not happen); the other is the candidate exit
+ // instruction.
+ if (Inc0 == Phi)
+ Chain = Inc1;
+ else if (Inc1 == Phi)
+ Chain = Inc0;
+ else
+ return false;
+ } else {
+ // Same idea, but use the select's true/false operands.
+ auto *ExitSelect = cast<SelectInst>(Exit);
+ Value *TrueVal = ExitSelect->getTrueValue();
+ Value *FalseVal = ExitSelect->getFalseValue();
+
+ if (TrueVal == Phi)
+ Chain = dyn_cast<Instruction>(FalseVal);
+ else if (FalseVal == Phi)
+ Chain = dyn_cast<Instruction>(TrueVal);
+ else
+ return false;
+ }
+
+ // The candidate must be the tracked exact-FP instruction, and must not
+ // be used anywhere else, or replacing it with a masked operation later
+ // would not be equivalent.
+ if (!Chain || Chain != ExactFPMathInst || !Chain->hasOneUse())
+ return false;
+
+ Exit = Chain;
+ }
+
if (Kind == RecurKind::FAdd && Exit->getOpcode() != Instruction::FAdd)
return false;
@@ -967,9 +1012,12 @@ RecurrenceDescriptor::isConditionalRdxPattern(Instruction *I) {
return InstDesc(false, I);
Value *Op1, *Op2;
- if (!(((m_FAdd(m_Value(Op1), m_Value(Op2)).match(I1) ||
- m_FSub(m_Value(Op1), m_Value(Op2)).match(I1)) &&
- I1->isFast()) ||
+ // FAdd/FSub don't need fast-math flags here: without them, the reduction
+ // may still be vectorized later as an ordered reduction, see
+ // checkOrderedReduction.
+ bool IsFAddOrFSub = m_FAdd(m_Value(Op1), m_Value(Op2)).match(I1) ||
+ m_FSub(m_Value(Op1), m_Value(Op2)).match(I1);
+ if (!(IsFAddOrFSub ||
(m_FMul(m_Value(Op1), m_Value(Op2)).match(I1) && (I1->isFast())) ||
((m_Add(m_Value(Op1), m_Value(Op2)).match(I1) ||
m_Sub(m_Value(Op1), m_Value(Op2)).match(I1))) ||
@@ -981,6 +1029,11 @@ RecurrenceDescriptor::isConditionalRdxPattern(Instruction *I) {
if (!IPhi || IPhi != FalseVal)
return InstDesc(false, I);
+ // Without reassoc, I1 is exact FP math: only allow vectorizing this
+ // reduction if it is later recognized as an ordered reduction.
+ if (IsFAddOrFSub && !I1->hasAllowReassoc())
+ return InstDesc(true, I, I1);
+
return InstDesc(true, I);
}
@@ -1284,20 +1337,24 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
// to check for a pair of icmp/select, for which we use getNextInstruction and
// isCorrectOpcode functions to step the right number of instruction, and
// check the icmp/select pair.
- // FIXME: We also do not attempt to look through Select's yet, which might
- // be part of the reduction chain, or attempt to looks through And's to find a
- // smaller bitwidth. Subs are also currently not allowed (which are usually
- // treated as part of a add reduction) as they are expected to generally be
- // more expensive than out-of-loop reductions, and need to be costed more
+ // FIXME: We also do not attempt to look through And's to find a smaller
+ // bitwidth. Subs are also currently not allowed (which are usually treated
+ // as part of a add reduction) as they are expected to generally be more
+ // expensive than out-of-loop reductions, and need to be costed more
// carefully.
unsigned ExpectedUses = 1;
if (IsMinMax)
ExpectedUses = 2;
+ // The phi or select that blends a conditional reduction's updated and
+ // unchanged values, if any; skipped below like a phi since it isn't part
+ // of the fadd/fmuladd chain itself.
+ Instruction *Blend = nullptr;
+
auto getNextInstruction = [&](Instruction *Cur) -> Instruction * {
for (auto *User : Cur->users()) {
Instruction *UI = cast<Instruction>(User);
- if (isa<PHINode>(UI))
+ if (isa<PHINode>(UI) || UI == Blend)
continue;
if (IsMinMax) {
// We are expecting a icmp/select pair, which we go to the next select
@@ -1331,26 +1388,49 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
return Cur->getOpcode() == getOpcode();
};
- // Attempt to look through Phis which are part of the reduction chain
+ // Attempt to look through a phi or select blending in a conditional
+ // reduction's unchanged value, which is part of the reduction chain.
unsigned ExtraPhiUses = 0;
Instruction *RdxInstr = LoopExitInstr;
- if (auto ExitPhi = dyn_cast<PHINode>(LoopExitInstr)) {
- if (ExitPhi->getNumIncomingValues() != 2)
- return {};
+ if (isa<PHINode>(LoopExitInstr) ||
+ (!IsMinMax && isa<SelectInst>(LoopExitInstr))) {
+ Instruction *Chain = nullptr;
+ if (auto *ExitPhi = dyn_cast<PHINode>(LoopExitInstr)) {
+ if (ExitPhi->getNumIncomingValues() != 2)
+ return {};
- Instruction *Inc0 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(0));
- Instruction *Inc1 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(1));
+ Instruction *Inc0 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(0));
+ Instruction *Inc1 = dyn_cast<Instruction>(ExitPhi->getIncomingValue(1));
- Instruction *Chain = nullptr;
- if (Inc0 == Phi)
- Chain = Inc1;
- else if (Inc1 == Phi)
- Chain = Inc0;
- else
+ if (Inc0 == Phi)
+ Chain = Inc1;
+ else if (Inc1 == Phi)
+ Chain = Inc0;
+ else
+ return {};
+ } else {
+ // Same idea, but use the select's true/false operands. Min/max
+ // reductions are excluded above since their select IS the reduction
+ // operation itself (handled by isCorrectOpcode/getNextInstruction
+ // below), not a wrapper around one.
+ auto *ExitSelect = cast<SelectInst>(LoopExitInstr);
+ Value *TrueVal = ExitSelect->getTrueValue();
+ Value *FalseVal = ExitSelect->getFalseValue();
+
+ if (TrueVal == Phi)
+ Chain = dyn_cast<Instruction>(FalseVal);
+ else if (FalseVal == Phi)
+ Chain = dyn_cast<Instruction>(TrueVal);
+ else
+ return {};
+ }
+
+ if (!Chain)
return {};
RdxInstr = Chain;
ExtraPhiUses = 1;
+ Blend = LoopExitInstr;
}
// The loop exit instruction we check first (as a quick test) but add last. We
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 459e8f86cb337..bdeda6d5399dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1096,6 +1096,23 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
continue;
}
+ // Some conditional reductions use a select instead of a branch, e.g.
+ // `select(cond, phi + x, phi)`. Fold it away like a blend, since the
+ // reduction recipe created below already applies the condition as a
+ // mask. But a select isn't a phi, so we also erase it here - otherwise
+ // it would be left in the block pointing at the reduction recipe,
+ // which only gets created afterwards.
+ if (!RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind) &&
+ match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue()))) {
+ VPValue *TrueVal = CurrentLink->getOperand(1);
+ VPValue *FalseVal = CurrentLink->getOperand(2);
+ if (TrueVal == PhiR || FalseVal == PhiR) {
+ CurrentLink->replaceAllUsesWith(TrueVal == PhiR ? FalseVal : TrueVal);
+ ToDelete.push_back(CurrentLink);
+ continue;
+ }
+ }
+
if (IsFPRecurrence) {
FastMathFlags CurFMF =
cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlagsOrNone();
@@ -1166,6 +1183,32 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
assert(PhiR->getVFScaleFactor() == 1 &&
"inloop reductions must be unscaled");
VPValue *CondOp = cast<VPInstruction>(CurrentLink)->getMask();
+ if (!RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
+ // A select wrapping this fadd/fsub (e.g. `select(cond, phi + x,
+ // phi)`) carries the reduction's real condition. Any mask already
+ // on the fadd/fsub just means it runs unconditionally, so we AND
+ // the select's condition onto the loop's header mask instead --
+ // that's the shape a later EVL-lowering pass expects to see.
+ for (VPUser *U : CurrentLink->users()) {
+ auto *Sel = dyn_cast<VPInstruction>(U);
+ if (!Sel ||
+ !match(Sel, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ continue;
+ VPValue *TrueVal = Sel->getOperand(1);
+ VPValue *FalseVal = Sel->getOperand(2);
+ if ((TrueVal == CurrentLink && FalseVal == PreviousLink) ||
+ (FalseVal == CurrentLink && TrueVal == PreviousLink)) {
+ VPValue *SelCond = Sel->getOperand(0);
+ if (VPValue *HeaderMask =
+ Plan.getVectorLoopRegion()->getHeaderMask()) {
+ VPBuilder CondBuilder(LinkVPBB, CurrentLink->getIterator());
+ SelCond = CondBuilder.createLogicalAnd(HeaderMask, SelCond);
+ }
+ CondOp = SelCond;
+ break;
+ }
+ }
+ }
auto *RedRecipe = new VPReductionRecipe(
Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
getReductionStyle(/*IsInLoop=*/true, PhiR->isOrdered(), 1),
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
index f3bd08a652bb5..b5b75d819a5f0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
@@ -89,5 +89,201 @@ for.end:
ret float %add
}
+define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %c, i64 %n) {
+; IF-EVL-LABEL: @cond_fadd(
+; IF-EVL-NEXT: entry:
+; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
+; IF-EVL: vector.body:
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
+; IF-EVL-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP4]] = call float @llvm.vp.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[VP_OP_LOAD1]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]]
+; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
+; IF-EVL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; IF-EVL-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL: middle.block:
+; IF-EVL-NEXT: br label [[FOR_END:%.*]]
+; IF-EVL: for.end:
+; IF-EVL-NEXT: ret float [[TMP4]]
+;
+; NO-VP-LABEL: @cond_fadd(
+; NO-VP-NEXT: entry:
+; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP: vector.ph:
+; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
+; NO-VP: vector.body:
+; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; NO-VP-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; NO-VP-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[TMP4]], <vscale x 4 x float> poison)
+; NO-VP-NEXT: [[TMP6:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float -0.000000e+00)
+; NO-VP-NEXT: [[TMP7]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP6]])
+; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; NO-VP-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; NO-VP: middle.block:
+; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; NO-VP: scalar.ph:
+; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; NO-VP-NEXT: br label [[FOR_BODY:%.*]]
+; NO-VP: for.body:
+; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; NO-VP-NEXT: [[SUM_010:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_1:%.*]], [[FOR_INC]] ]
+; NO-VP-NEXT: [[ARRAYIDX_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX_C]], align 4
+; NO-VP-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP9]], 0
+; NO-VP-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; NO-VP: if.then:
+; NO-VP-NEXT: [[ARRAYIDX_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX_A]], align 4
+; NO-VP-NEXT: [[ADD:%.*]] = fadd float [[SUM_010]], [[TMP10]]
+; NO-VP-NEXT: br label [[FOR_INC]]
+; NO-VP: for.inc:
+; NO-VP-NEXT: [[SUM_1]] = phi float [ [[SUM_010]], [[FOR_BODY]] ], [ [[ADD]], [[IF_THEN]] ]
+; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; NO-VP: for.end:
+; NO-VP-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], [[FOR_INC]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT: ret float [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum.010 = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ %0 = load i32, ptr %arrayidx.c, align 4
+ %tobool.not = icmp eq i32 %0, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx.a = getelementptr inbounds float, ptr %a, i64 %iv
+ %1 = load float, ptr %arrayidx.a, align 4
+ %add = fadd float %sum.010, %1
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi float [ %sum.010, %for.body ], [ %add, %if.then ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2
+
+for.end:
+ %sum.1.lcssa = phi float [ %sum.1, %for.inc ]
+ ret float %sum.1.lcssa
+}
+
+define float @cond_fadd_select(ptr noalias nocapture readonly %a, i64 %n) {
+; IF-EVL-LABEL: @cond_fadd_select(
+; IF-EVL-NEXT: entry:
+; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
+; IF-EVL: vector.body:
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP2:%.*]] = fcmp ogt <vscale x 4 x float> [[VP_OP_LOAD]], zeroinitializer
+; IF-EVL-NEXT: [[TMP3]] = call float @llvm.vp.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; IF-EVL-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; IF-EVL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; IF-EVL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IF-EVL: middle.block:
+; IF-EVL-NEXT: br label [[FOR_END:%.*]]
+; IF-EVL: for.end:
+; IF-EVL-NEXT: ret float [[TMP3]]
+;
+; NO-VP-LABEL: @cond_fadd_select(
+; NO-VP-NEXT: entry:
+; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP: vector.ph:
+; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
+; NO-VP: vector.body:
+; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; NO-VP-NEXT: [[TMP4:%.*]] = fcmp ogt <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
+; NO-VP-NEXT: [[TMP5:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> splat (float -0.000000e+00)
+; NO-VP-NEXT: [[TMP6]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP5]])
+; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; NO-VP-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; NO-VP: middle.block:
+; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; NO-VP: scalar.ph:
+; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; NO-VP-NEXT: br label [[FOR_BODY:%.*]]
+; NO-VP: for.body:
+; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT: [[SUM_09:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_1:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; NO-VP-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP8]], 0.000000e+00
+; NO-VP-NEXT: [[ADD:%.*]] = fadd float [[SUM_09]], [[TMP8]]
+; NO-VP-NEXT: [[SUM_1]] = select i1 [[CMP1]], float [[ADD]], float [[SUM_09]]
+; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; NO-VP: for.end:
+; NO-VP-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT: ret float [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.09 = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+ %0 = load float, ptr %arrayidx, align 4
+ %cmp1 = fcmp ogt float %0, 0.000000e+00
+ %add = fadd float %sum.09, %0
+ %sum.1 = select i1 %cmp1, float %add, float %sum.09
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:
+ %sum.1.lcssa = phi float [ %sum.1, %for.body ]
+ ret float %sum.1.lcssa
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable"}
+!2 = distinct !{!2, !1}
+!4 = distinct !{!4, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index 75ded5e7bcf70..24260b429ccd7 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -659,20 +659,44 @@ define float @fcmp_0_fsub_select1_novectorize(ptr noalias %x, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]
; CHECK: [[FOR_HEADER]]:
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[ZEXT]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[TMP7]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = fsub <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP4]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP0]], 0.000000e+00
; CHECK-NEXT: [[SUB:%.*]] = fsub float [[SUM_1]], [[TMP0]]
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[SUB]], float [[SUM_1]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
-; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY1]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]
@@ -739,7 +763,7 @@ define double @fcmp_0_fsub_select2(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -758,7 +782,7 @@ define double @fcmp_0_fsub_select2(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[SUB]], double [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -810,20 +834,44 @@ define double @fcmp_0_fsub_select2_notvectorize(ptr noalias %x, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]
; CHECK: [[FOR_HEADER]]:
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[ZEXT]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[TMP7]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ <double 0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x double> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = fsub <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x double> [[TMP3]], <4 x double> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP4]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[ARRAYIDX1]], align 4
; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt double [[TMP0]], 0.000000e+00
; CHECK-NEXT: [[SUB:%.*]] = fsub double [[SUM_1]], [[TMP0]]
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[SUB]], double [[SUM_1]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
-; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY1]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]
@@ -890,7 +938,7 @@ define float @fcmp_0_fmult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP4]], <4 x float> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -909,7 +957,7 @@ define float @fcmp_0_fmult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[MULT]], float [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1041,7 +1089,7 @@ define double @fcmp_0_fmult_select2(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fmul.v4f64(double 1.000000e+00, <4 x double> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -1060,7 +1108,7 @@ define double @fcmp_0_fmult_select2(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[MULT]], double [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1202,7 +1250,7 @@ define float @fcmp_multi(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: [[TMP10]] = fadd fast <4 x float> [[PREDPHI1]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP10]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
@@ -1232,7 +1280,7 @@ define float @fcmp_multi(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: [[SUM_1]] = fadd fast float [[DOTPN]], [[SUM_011]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1323,7 +1371,7 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
@@ -1352,7 +1400,7 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[SUB]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1576,7 +1624,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1595,7 +1643,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i64 [[ADD]], i64 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i64 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1656,7 +1704,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -1673,7 +1721,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[SUB]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = sub nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1730,7 +1778,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -1749,7 +1797,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[MULT]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1815,4 +1863,8 @@ for.end:
; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}
; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}
; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]}
+; CHECK: [[LOOP32]] = distinct !{[[LOOP32]], [[META1]], [[META2]]}
+; CHECK: [[LOOP33]] = distinct !{[[LOOP33]], [[META2]], [[META1]]}
+; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META1]], [[META2]]}
+; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[META2]], [[META1]]}
;.
More information about the llvm-commits
mailing list