[llvm] [LV] Handle FSub Partial Reductions (PR #191186)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 03:52:49 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/191186
>From 42d70fe49c641fb7ca5599ff7e2f44c704c76ce5 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 30 Mar 2026 15:28:54 +0000
Subject: [PATCH 01/19] [LV] Handle FSub Partial Reductions
Introduces a new RecurKind value 'FSub' in order to handle partial
reductions of floating point values.
This is done by following the existing method for integer partial
reductions, doing a positive accumulation followed by a final
subtraction in the middle block.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 1 +
llvm/lib/Analysis/IVDescriptors.cpp | 13 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 4 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 2 +
.../Transforms/Vectorize/LoopVectorize.cpp | 27 ++-
.../Transforms/Vectorize/VPlanPatternMatch.h | 30 ++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 +-
.../partial-reduce-sub-epilogue-vec.ll | 169 ++++++++++++++++++
.../AArch64/partial-reduce-sub.ll | 114 ++++++++++++
10 files changed, 356 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 8383da0f9e879..8cfb8a78a9a96 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -46,6 +46,7 @@ enum class RecurKind {
UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
FAdd, ///< Sum of floats.
+ FSub, ///< Subtraction of floats.
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index fa92ef6d8cb7c..f17ca3178376d 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1007,13 +1007,16 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
return InstDesc(Kind == RecurKind::FMul, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::FSub:
+ return InstDesc(Kind == RecurKind::FSub, I,
+ I->hasAllowReassoc() ? nullptr : I);
case Instruction::FAdd:
return InstDesc(Kind == RecurKind::FAdd, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::Select:
- if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
- Kind == RecurKind::Add || Kind == RecurKind::Mul ||
- Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs)
+ if (Kind == RecurKind::FAdd || Kind == RecurKind::FSub ||
+ Kind == RecurKind::FMul || Kind == RecurKind::Add ||
+ Kind == RecurKind::Mul || Kind == RecurKind::Sub ||
+ Kind == RecurKind::AddChainWithSubs)
return isConditionalRdxPattern(I);
if (isFindRecurrenceKind(Kind) && SE)
return isFindPattern(L, OrigPhi, I, *SE);
@@ -1102,6 +1105,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
return true;
}
+ if (AddReductionVar(Phi, RecurKind::FSub, TheLoop, RedDes, DB, AC, DT, SE)) {
+ LLVM_DEBUG(dbgs() << "Found an FSub reduction PHI." << *Phi << "\n");
+ return true;
+ }
if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, RedDes, DB, AC, DT, SE)) {
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index aff89e00523c0..ff35a540d7662 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5978,7 +5978,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
return Invalid;
if ((Opcode != Instruction::Add && Opcode != Instruction::Sub &&
- Opcode != Instruction::FAdd) ||
+ Opcode != Instruction::FAdd && Opcode != Instruction::FSub) ||
OpAExtend == TTI::PR_None)
return Invalid;
@@ -6045,7 +6045,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
NEONPred);
};
- bool IsSub = Opcode == Instruction::Sub;
+ bool IsSub = (Opcode == Instruction::Sub) || (Opcode == Instruction::FSub);
InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
if (AccumLT.second.getScalarType() == MVT::i32 &&
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 9c7d10629adf6..f7bebf08c0d3b 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1097,6 +1097,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::Xor:
return Intrinsic::vector_reduce_xor;
case RecurKind::FMulAdd:
+ case RecurKind::FSub:
case RecurKind::FAdd:
return Intrinsic::vector_reduce_fadd;
case RecurKind::FMul:
@@ -1554,6 +1555,7 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::FMaximumNum:
return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
case RecurKind::FMulAdd:
+ case RecurKind::FSub:
case RecurKind::FAdd:
return Builder.CreateFAddReduce(getIdentity(), Src);
case RecurKind::FMul:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6e7a17c8b59d5..696e9949c270a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8034,17 +8034,30 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
// Partial sub-reductions always start at 0 and account for the
// reduction start value in a final subtraction. Update it to use the
// resume value from the main vector loop.
- if (PhiR->getVFScaleFactor() > 1 &&
- PhiR->getRecurrenceKind() == RecurKind::Sub) {
+ if ((PhiR->getVFScaleFactor() > 1 &&
+ PhiR->getRecurrenceKind() == RecurKind::Sub) ||
+ PhiR->getRecurrenceKind() == RecurKind::FSub) {
auto *Sub = cast<VPInstruction>(RdxResult->getSingleUser());
- assert(Sub->getOpcode() == Instruction::Sub && "Unexpected opcode");
+ assert((Sub->getOpcode() == Instruction::Sub ||
+ Sub->getOpcode() == Instruction::FSub) &&
+ "Unexpected opcode");
assert(isa<VPIRValue>(Sub->getOperand(0)) &&
"Expected operand to match the original start value of the "
"reduction");
- assert(VPlanPatternMatch::match(VPI->getOperand(0),
- VPlanPatternMatch::m_ZeroInt()) &&
- "Expected start value for partial sub-reduction to start at "
- "zero");
+ // For integer sub-reductions, verify start value is zero.
+ // For FP sub-reductions, verify start value is negative zero.
+ if (PhiR->getRecurrenceKind() == RecurKind::Sub) {
+ assert(VPlanPatternMatch::match(VPI->getOperand(0),
+ VPlanPatternMatch::m_ZeroInt()) &&
+ "Expected start value for partial sub-reduction to start "
+ "at zero");
+ } else if (PhiR->getRecurrenceKind() == RecurKind::FSub) {
+ assert(
+ VPlanPatternMatch::match(VPI->getOperand(0),
+ VPlanPatternMatch::m_NegZeroFP()) &&
+ "Expected start value for partial sub-reduction to start "
+ "at negative zero");
+ }
Sub->setOperand(0, StartVal);
} else
VPI->setOperand(0, StartVal);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 242046480f6e9..6b6f46d3282c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -152,6 +152,36 @@ inline int_pred_ty<is_zero_int> m_ZeroInt() {
/// For vectors, this includes constants with undefined elements.
inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
+/// Match a floating-point constant if Pred::isValue returns true for the
+/// APFloat.
+template <typename Pred> struct fp_pred_ty {
+ Pred P;
+
+ fp_pred_ty(Pred P) : P(std::move(P)) {}
+ fp_pred_ty() : P() {}
+
+ bool match(const VPValue *VPV) const {
+ auto *VPI = dyn_cast<VPInstruction>(VPV);
+ if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
+ VPV = VPI->getOperand(0);
+ auto *VPIRV = dyn_cast<VPIRValue>(VPV);
+ if (!VPIRV)
+ return false;
+ auto *CFP = dyn_cast<ConstantFP>(VPIRV->getValue());
+ if (!CFP)
+ return false;
+ return P.isValue(CFP->getValueAPF());
+ }
+};
+struct is_neg_zero_fp {
+ bool isValue(const APFloat &C) const { return C.isNegZero(); }
+};
+
+/// Match a floating-point negative zero.
+inline fp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
+ return fp_pred_ty<is_neg_zero_fp>();
+}
+
struct bind_apint {
const APInt *&Res;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 25fe37124b017..a4678fe6098b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -762,8 +762,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
// For sub-recurrences, each part's reduction variable is already
// negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
Instruction::BinaryOps Opcode =
- RK == RecurKind::Sub
- ? Instruction::Add
+ (RK == RecurKind::Sub) ? Instruction::Add
+ : (RK == RecurKind::FSub)
+ ? Instruction::FAdd
: (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
ReducedPartRdx =
Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7eab86e8fd7ca..10542570ab0e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6016,7 +6016,7 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// If this is the last value in a sub-reduction chain, then update the PHI
// node to start at `0` and update the reduction-result to subtract from
// the PHI's start value.
- if (Chain.RK != RecurKind::Sub)
+ if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
return;
VPValue *OldStartValue = StartInst->getOperand(0);
@@ -6027,7 +6027,8 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
assert(RdxResult && "Could not find reduction result");
VPBuilder Builder = VPBuilder::getToInsertAfter(RdxResult);
- constexpr unsigned SubOpc = Instruction::BinaryOps::Sub;
+ unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
+ : Instruction::BinaryOps::Sub;
VPInstruction *NewResult = Builder.createNaryOp(
SubOpc, {OldStartValue, RdxResult}, VPIRFlags::getDefaultFlags(SubOpc),
RdxPhi->getDebugLoc());
@@ -6127,7 +6128,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
// optimizeExtendsForPartialReduction.
Op = CastSource;
} else if (UpdateR->getOpcode() == Instruction::Add ||
- UpdateR->getOpcode() == Instruction::FAdd) {
+ UpdateR->getOpcode() == Instruction::FAdd ||
+ UpdateR->getOpcode() == Instruction::FSub) {
// Match: UpdateR(PrevValue, ext(...))
// TODO: Remove the add/fadd restriction (we should be able to handle this
// case for sub reductions too).
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index 8db3a3294d7bc..b210f8b49f2e3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -185,4 +185,173 @@ exit:
ret i32 %sub
}
+define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
+; CHECK-EPI-LABEL: define float @fsub_reduction(
+; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
+; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-EPI: [[VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
+; CHECK-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x half>, ptr [[TMP8]], align 4
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x half>, ptr [[TMP9]], align 4
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = fpext <4 x half> [[WIDE_LOAD4]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = fpext <4 x half> [[WIDE_LOAD5]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <4 x float> [[TMP10]], [[TMP11]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI3]], <4 x float> [[TMP12]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 36
+; CHECK-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE6]])
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-EPI: [[LOOP]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-EPI: [[EXIT]]:
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+; CHECK-PARTIAL-RED-EPI-LABEL: define float @fsub_reduction(
+; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP8]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x half>, ptr [[TMP9]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = fpext <8 x half> [[WIDE_LOAD5]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <8 x float> [[TMP10]], [[TMP11]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI3]], <8 x float> [[TMP12]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE6]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi float [ %startval, %entry ], [ %sub, %loop ]
+ %src1.gep = getelementptr half, ptr %src1, i32 %iv
+ %src1.load = load half, ptr %src1.gep, align 4
+ %src1.load.ext = fpext half %src1.load to float
+ %src2.gep = getelementptr half, ptr %src2, i32 %iv
+ %src2.load = load half, ptr %src2.gep, align 4
+ %src2.load.ext = fpext half %src2.load to float
+ %mul = fmul reassoc contract float %src1.load.ext, %src2.load.ext
+ %sub = fsub reassoc contract float %accum, %mul
+ %iv.next = add i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv, 38
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret float %sub
+}
+
+
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index 2aea67f6e5499..f0363fe62d1ad 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -269,6 +269,120 @@ exit:
ret i64 %sub
}
+define float @fdotp_fsub(ptr %a, ptr %b) #0 {
+; CHECK-INTERLEAVE1-LABEL: define float @fdotp_fsub(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: entry:
+; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1: vector.ph:
+; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1: vector.body:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1: middle.block:
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVE1: for.exit:
+; CHECK-INTERLEAVE1-NEXT: ret float [[TMP7]]
+;
+; CHECK-INTERLEAVED-LABEL: define float @fdotp_fsub(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: entry:
+; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED: vector.ph:
+; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED: vector.body:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[TMP0]], i64 8
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[TMP2]], i64 8
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x half>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD3]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fmul reassoc contract <8 x float> [[TMP4]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP6]])
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = fmul reassoc contract <8 x float> [[TMP7]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE5]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI1]], <8 x float> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED: middle.block:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <4 x float> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = fsub float 0.000000e+00, [[TMP11]]
+; CHECK-INTERLEAVED-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVED: for.exit:
+; CHECK-INTERLEAVED-NEXT: ret float [[TMP12]]
+;
+; CHECK-MAXBW-LABEL: define float @fdotp_fsub(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: entry:
+; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW: vector.ph:
+; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW: vector.body:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-MAXBW-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW: middle.block:
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]]
+; CHECK-MAXBW-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-MAXBW: for.exit:
+; CHECK-MAXBW-NEXT: ret float [[TMP7]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi float [ 0.0, %entry ], [ %sub, %for.body ]
+ %gep.a = getelementptr half, ptr %a, i64 %iv
+ %load.a = load half, ptr %gep.a, align 2
+ %ext.a = fpext half %load.a to float
+ %gep.b = getelementptr half, ptr %b, i64 %iv
+ %load.b = load half, ptr %gep.b, align 2
+ %ext.b = fpext half %load.b to float
+ %mul = fmul reassoc contract float %ext.b, %ext.a
+ %sub = fsub reassoc contract float %accum, %mul
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:
+ ret float %sub
+}
+
!7 = distinct !{!7, !8, !9, !10}
!8 = !{!"llvm.loop.mustprogress"}
!9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
>From d6810845a0ac0debcf4b4b029c9650802b9a3a57 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 14 Apr 2026 09:48:41 +0000
Subject: [PATCH 02/19] Add missing RecurKind::FSub cases in SLPVectorizer
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3a6b3dd28b73b..22e4488200ef5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29187,6 +29187,7 @@ class HorizontalReduction {
// res = vv
break;
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -29338,6 +29339,7 @@ class HorizontalReduction {
// res = vv
return VectorizedValue;
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -29441,6 +29443,7 @@ class HorizontalReduction {
return Builder.CreateFMul(VectorizedValue, Scale);
}
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
>From 66cde3861ec70a0a8f72fa9fb7e48e7f6dc1267f Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 15 Apr 2026 13:49:20 +0000
Subject: [PATCH 03/19] Add missing RecurKind::FSub case and update failing
tests
---
llvm/lib/Analysis/IVDescriptors.cpp | 2 +
.../partial-reduce-sub-epilogue-vec.ll | 80 +++----------------
.../Transforms/LoopVectorize/if-reduction.ll | 50 +++---------
3 files changed, 24 insertions(+), 108 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index f17ca3178376d..dc2bf08240f7a 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1231,6 +1231,8 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::FMulAdd:
case RecurKind::FAdd:
return Instruction::FAdd;
+ case RecurKind::FSub:
+ return Instruction::FSub;
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index b210f8b49f2e3..2d24d02732b6d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -188,10 +188,8 @@ exit:
define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-LABEL: define float @fsub_reduction(
; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
-; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*:]]
+; CHECK-EPI-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-EPI: [[VECTOR_PH]]:
; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-EPI: [[VECTOR_BODY]]:
@@ -211,38 +209,12 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI: [[MIDDLE_BLOCK]]:
; CHECK-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
; CHECK-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
-; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
-; CHECK-EPI: [[VEC_EPILOG_PH]]:
-; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x half>, ptr [[TMP8]], align 4
-; CHECK-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x half>, ptr [[TMP9]], align 4
-; CHECK-EPI-NEXT: [[TMP10:%.*]] = fpext <4 x half> [[WIDE_LOAD4]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP11:%.*]] = fpext <4 x half> [[WIDE_LOAD5]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <4 x float> [[TMP10]], [[TMP11]]
-; CHECK-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI3]], <4 x float> [[TMP12]])
-; CHECK-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 36
-; CHECK-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE6]])
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
-; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-EPI: [[LOOP]]:
-; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
@@ -253,17 +225,15 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
-; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-EPI: [[EXIT]]:
-; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ]
; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
;
; CHECK-PARTIAL-RED-EPI-LABEL: define float @fsub_reduction(
; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*:]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
@@ -283,38 +253,12 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP8]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x half>, ptr [[TMP9]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = fpext <8 x half> [[WIDE_LOAD5]] to <8 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <8 x float> [[TMP10]], [[TMP11]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI3]], <8 x float> [[TMP12]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE6]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
@@ -325,9 +269,9 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index 40c5e7f937166..fe35d30d83e89 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1303,38 +1303,10 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
-; CHECK-NEXT: [[TMP6:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP12]], 1.000000e+00
@@ -1352,9 +1324,9 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[SUB]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]
; CHECK: [[FOR_END_LOOPEXIT]]:
-; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ]
; CHECK-NEXT: br label %[[FOR_END]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]
@@ -1576,7 +1548,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1595,7 +1567,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i64 [[ADD]], i64 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i64 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1656,7 +1628,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -1673,7 +1645,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[SUB]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = sub nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1730,7 +1702,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -1749,7 +1721,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[MULT]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1813,6 +1785,4 @@ for.end:
; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]}
; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}
; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}
-; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}
-; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]}
;.
>From b537773a2af42f576462507bb79abad552eafe4d Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 16 Apr 2026 12:58:01 +0000
Subject: [PATCH 04/19] Refactor start value check for epilogue loops
---
.../Transforms/Vectorize/LoopVectorize.cpp | 22 +++++++-------
.../Transforms/Vectorize/VPlanPatternMatch.h | 30 -------------------
2 files changed, 10 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 696e9949c270a..89811b5332f70 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8046,18 +8046,16 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
"reduction");
// For integer sub-reductions, verify start value is zero.
// For FP sub-reductions, verify start value is negative zero.
- if (PhiR->getRecurrenceKind() == RecurKind::Sub) {
- assert(VPlanPatternMatch::match(VPI->getOperand(0),
- VPlanPatternMatch::m_ZeroInt()) &&
- "Expected start value for partial sub-reduction to start "
- "at zero");
- } else if (PhiR->getRecurrenceKind() == RecurKind::FSub) {
- assert(
- VPlanPatternMatch::match(VPI->getOperand(0),
- VPlanPatternMatch::m_NegZeroFP()) &&
- "Expected start value for partial sub-reduction to start "
- "at negative zero");
- }
+ auto StartValueIsIdentity = [&] {
+ Value *IdentityValue = getRecurrenceIdentity(
+ PhiR->getRecurrenceKind(), ResumeV->getType(), {});
+ auto *StartValue = dyn_cast<VPIRValue>(VPI->getOperand(0));
+ return StartValue && StartValue->getValue() == IdentityValue;
+ };
+ assert(StartValueIsIdentity() &&
+ "Expected start value for partial sub-reduction to be zero "
+ "(or negative zero)");
+
Sub->setOperand(0, StartVal);
} else
VPI->setOperand(0, StartVal);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 6b6f46d3282c9..242046480f6e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -152,36 +152,6 @@ inline int_pred_ty<is_zero_int> m_ZeroInt() {
/// For vectors, this includes constants with undefined elements.
inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
-/// Match a floating-point constant if Pred::isValue returns true for the
-/// APFloat.
-template <typename Pred> struct fp_pred_ty {
- Pred P;
-
- fp_pred_ty(Pred P) : P(std::move(P)) {}
- fp_pred_ty() : P() {}
-
- bool match(const VPValue *VPV) const {
- auto *VPI = dyn_cast<VPInstruction>(VPV);
- if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
- VPV = VPI->getOperand(0);
- auto *VPIRV = dyn_cast<VPIRValue>(VPV);
- if (!VPIRV)
- return false;
- auto *CFP = dyn_cast<ConstantFP>(VPIRV->getValue());
- if (!CFP)
- return false;
- return P.isValue(CFP->getValueAPF());
- }
-};
-struct is_neg_zero_fp {
- bool isValue(const APFloat &C) const { return C.isNegZero(); }
-};
-
-/// Match a floating-point negative zero.
-inline fp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
- return fp_pred_ty<is_neg_zero_fp>();
-}
-
struct bind_apint {
const APInt *&Res;
>From 273a0049842e95c19157d55bb53e5316c51aa2e7 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 16 Apr 2026 15:18:27 +0000
Subject: [PATCH 05/19] Maintain fallthrough to RecurKind:FAdd for mixed
fadd/fsub reductions
---
llvm/lib/Analysis/IVDescriptors.cpp | 2 +-
.../Transforms/LoopVectorize/if-reduction.ll | 54 ++++++++++++++-----
2 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index dc2bf08240f7a..be7b6ec0738b9 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1007,7 +1007,7 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
return InstDesc(Kind == RecurKind::FMul, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::FSub:
- return InstDesc(Kind == RecurKind::FSub, I,
+ return InstDesc(Kind == RecurKind::FSub || Kind == RecurKind::FAdd, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::FAdd:
return InstDesc(Kind == RecurKind::FAdd, I,
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index fe35d30d83e89..906f0e1d9e7f6 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1303,12 +1303,40 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
+; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP1]], <4 x float> [[PREDPHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV1]]
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP12]], 1.000000e+00
; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
; CHECK: [[IF_THEN]]:
@@ -1322,11 +1350,11 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: br label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[SUB]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
-; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ]
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]
@@ -1548,7 +1576,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1567,7 +1595,7 @@ define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i64 [[ADD]], i64 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i64 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1628,7 +1656,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -1645,7 +1673,7 @@ define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[SUB]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = sub nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1702,7 +1730,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]
@@ -1721,7 +1749,7 @@ define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) {
; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[MULT]], i32 [[SUM_1]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
@@ -1785,4 +1813,6 @@ for.end:
; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]}
; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}
; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}
+; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}
+; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]}
;.
>From 75ad7b2f6e1842e73951e334d67c51cdeb854436 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 16 Apr 2026 15:47:11 +0000
Subject: [PATCH 06/19] revert variable names in if-reduction.ll
---
.../Transforms/LoopVectorize/if-reduction.ll | 52 +++++++++----------
1 file changed, 26 insertions(+), 26 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index 906f0e1d9e7f6..40c5e7f937166 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1308,35 +1308,35 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[TMP2:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
-; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP3]]
-; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP1]], <4 x float> [[PREDPHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
+; CHECK-NEXT: [[TMP6:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
+; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])
+; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
-; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV1]]
-; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP12]], 1.000000e+00
; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
; CHECK: [[IF_THEN]]:
@@ -1350,11 +1350,11 @@ define float @fcmp_fadd_fsub(ptr nocapture %a, i32 %n) {
; CHECK-NEXT: br label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[SUB]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[FOR_END_LOOPEXIT]]:
-; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label %[[FOR_END]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]
>From 06ed64f4a430b07c2fcc89fe0d7096fe947d52ad Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 17 Apr 2026 09:31:12 +0000
Subject: [PATCH 07/19] Add [[maybe_unused flag]] to StartValueIsIndentity
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 89811b5332f70..c50a4e0f17e8a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8046,7 +8046,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
"reduction");
// For integer sub-reductions, verify start value is zero.
// For FP sub-reductions, verify start value is negative zero.
- auto StartValueIsIdentity = [&] {
+ [[maybe_unused]] auto StartValueIsIdentity = [&] {
Value *IdentityValue = getRecurrenceIdentity(
PhiR->getRecurrenceKind(), ResumeV->getType(), {});
auto *StartValue = dyn_cast<VPIRValue>(VPI->getOperand(0));
>From b4fdae69f0f3a49543e388d5a268d484d8f79f22 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 20 Apr 2026 10:55:31 +0000
Subject: [PATCH 08/19] Enforce epilogue vector loop in fsub test.
---
.../partial-reduce-sub-epilogue-vec.ll | 145 ++++++++++++------
1 file changed, 101 insertions(+), 44 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index 2d24d02732b6d..ce2af3b221306 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -188,33 +188,61 @@ exit:
define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-LABEL: define float @fsub_reduction(
; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
-; CHECK-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*:]]
+; CHECK-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_VECTOR_BODY:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK-EPI: [[VECTOR_PH1]]:
; CHECK-EPI-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_PH]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH1]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_PH]] ]
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX1]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP0]], align 4
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX1]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x half>, ptr [[TMP1]], align 4
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = fpext <16 x half> [[WIDE_LOAD]] to <16 x float>
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = fpext <16 x half> [[WIDE_LOAD1]] to <16 x float>
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = fmul reassoc contract <16 x float> [[TMP5]], [[TMP10]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI]], <16 x float> [[TMP11]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT1]] = add nuw i32 [[INDEX1]], 16
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT1]], 32
+; CHECK-EPI-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK1]]:
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE]])
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_VECTOR_BODY]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-EPI: [[VECTOR_BODY]]:
-; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
-; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
-; CHECK-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x half>, ptr [[TMP8]], align 4
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x half>, ptr [[TMP9]], align 4
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD4]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD5]] to <4 x float>
; CHECK-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI3]], <4 x float> [[TMP4]])
; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
-; CHECK-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40
+; CHECK-EPI-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-EPI: [[MIDDLE_BLOCK]]:
-; CHECK-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
-; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE6]])
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-EPI: [[LOOP]]:
-; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
@@ -224,41 +252,69 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
-; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-EPI: [[EXIT]]:
-; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK1]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
;
; CHECK-PARTIAL-RED-EPI-LABEL: define float @fsub_reduction(
; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*:]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_VECTOR_BODY:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH1]]:
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_PH]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI1:%.*]] = phi <8 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH1]] ], [ [[PARTIAL_REDUCE1:%.*]], %[[VECTOR_PH]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX1]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP8]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX1]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x half>, ptr [[TMP9]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = fpext <16 x half> [[WIDE_LOAD2]] to <16 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = fpext <16 x half> [[WIDE_LOAD3]] to <16 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <16 x float> [[TMP10]], [[TMP11]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE1]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI1]], <16 x float> [[TMP12]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT1]] = add nuw i32 [[INDEX1]], 16
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT1]], 32
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK1]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE1]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_VECTOR_BODY]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
-; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_VECTOR_BODY]]
; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
@@ -268,10 +324,10 @@ define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK1]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
;
entry:
@@ -289,13 +345,14 @@ loop:
%mul = fmul reassoc contract float %src1.load.ext, %src2.load.ext
%sub = fsub reassoc contract float %accum, %mul
%iv.next = add i32 %iv, 1
- %exitcond.not = icmp eq i32 %iv, 38
- br i1 %exitcond.not, label %exit, label %loop
+ %exitcond.not = icmp eq i32 %iv, 40
+ br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !0
exit:
ret float %sub
}
-
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 16}
>From eea8767e41cf5d25c5e931459efe780b554afe3b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 20 Apr 2026 11:14:12 +0000
Subject: [PATCH 09/19] Add isSubRecurrenceKind check
---
llvm/include/llvm/Analysis/IVDescriptors.h | 3 +++
llvm/lib/Analysis/IVDescriptors.cpp | 4 ++++
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 +++---
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 8cfb8a78a9a96..39a85ed638d64 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -246,6 +246,9 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating point kind.
LLVM_ABI static bool isFloatingPointRecurrenceKind(RecurKind Kind);
+ /// Returns true if the recurrence kind is for a sub operation.
+ LLVM_ABI static bool isSubRecurrenceKind(RecurKind Kind);
+
/// Returns true if the recurrence kind is an integer min/max kind.
static bool isIntMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::UMin || Kind == RecurKind::UMax ||
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index be7b6ec0738b9..734b5b14cad68 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -92,6 +92,10 @@ static Instruction *lookThroughAnd(PHINode *Phi, Type *&RT,
return Phi;
}
+bool RecurrenceDescriptor::isSubRecurrenceKind(RecurKind Kind) {
+ return (Kind == RecurKind::Sub || Kind == RecurKind::FSub);
+}
+
/// Compute the minimal bit width needed to represent a reduction whose exit
/// instruction is given by Exit.
static std::pair<Type *, bool> computeRecurrenceType(Instruction *Exit,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c50a4e0f17e8a..c29de29ccf80c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8034,9 +8034,9 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
// Partial sub-reductions always start at 0 and account for the
// reduction start value in a final subtraction. Update it to use the
// resume value from the main vector loop.
- if ((PhiR->getVFScaleFactor() > 1 &&
- PhiR->getRecurrenceKind() == RecurKind::Sub) ||
- PhiR->getRecurrenceKind() == RecurKind::FSub) {
+ if (PhiR->getVFScaleFactor() > 1 &&
+ RecurrenceDescriptor::isSubRecurrenceKind(
+ PhiR->getRecurrenceKind())) {
auto *Sub = cast<VPInstruction>(RdxResult->getSingleUser());
assert((Sub->getOpcode() == Instruction::Sub ||
Sub->getOpcode() == Instruction::FSub) &&
>From 0b8f4cb157cc0ed010736fcbda4d1f536e11629c Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 21 Apr 2026 14:51:20 +0000
Subject: [PATCH 10/19] Introduce FAddChainWithSubs for mixed fp reductions
---
llvm/include/llvm/Analysis/IVDescriptors.h | 1 +
llvm/lib/Analysis/IVDescriptors.cpp | 24 +++++++++++++++----
.../AArch64/AArch64TargetTransformInfo.cpp | 1 +
llvm/lib/Transforms/Utils/LoopUtils.cpp | 2 ++
.../Transforms/Vectorize/SLPVectorizer.cpp | 3 +++
5 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 39a85ed638d64..c4d079bfba944 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -46,6 +46,7 @@ enum class RecurKind {
UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
FAdd, ///< Sum of floats.
+ FAddChainWithSubs, ///< A chain of fadds and fsubs.
FSub, ///< Subtraction of floats.
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 734b5b14cad68..c74cd9e0c7393 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1011,16 +1011,19 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
return InstDesc(Kind == RecurKind::FMul, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::FSub:
- return InstDesc(Kind == RecurKind::FSub || Kind == RecurKind::FAdd, I,
- I->hasAllowReassoc() ? nullptr : I);
+ return InstDesc(Kind == RecurKind::FSub ||
+ Kind == RecurKind::FAddChainWithSubs,
+ I, I->hasAllowReassoc() ? nullptr : I);
case Instruction::FAdd:
- return InstDesc(Kind == RecurKind::FAdd, I,
- I->hasAllowReassoc() ? nullptr : I);
+ return InstDesc(Kind == RecurKind::FAdd ||
+ Kind == RecurKind::FAddChainWithSubs,
+ I, I->hasAllowReassoc() ? nullptr : I);
case Instruction::Select:
if (Kind == RecurKind::FAdd || Kind == RecurKind::FSub ||
Kind == RecurKind::FMul || Kind == RecurKind::Add ||
Kind == RecurKind::Mul || Kind == RecurKind::Sub ||
- Kind == RecurKind::AddChainWithSubs)
+ Kind == RecurKind::AddChainWithSubs ||
+ Kind == RecurKind::FAddChainWithSubs)
return isConditionalRdxPattern(I);
if (isFindRecurrenceKind(Kind) && SE)
return isFindPattern(L, OrigPhi, I, *SE);
@@ -1117,6 +1120,12 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
}
+ if (AddReductionVar(Phi, RecurKind::FAddChainWithSubs, TheLoop, RedDes, DB,
+ AC, DT, SE)) {
+ LLVM_DEBUG(dbgs() << "Found a chained FADD-FSUB chained reduction PHI."
+ << *Phi << "\n");
+ return true;
+ }
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1233,6 +1242,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::FMul:
return Instruction::FMul;
case RecurKind::FMulAdd:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::FAdd:
return Instruction::FAdd;
case RecurKind::FSub:
@@ -1313,6 +1323,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
Kind == RecurKind::AddChainWithSubs)
return true;
+ if (Cur->getOpcode() == Instruction::FSub &&
+ Kind == RecurKind::AddChainWithSubs)
+ return true;
+
return Cur->getOpcode() == getOpcode();
};
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ff35a540d7662..6aa4987025edb 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5657,6 +5657,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
switch (RdxDesc.getRecurrenceKind()) {
case RecurKind::Sub:
case RecurKind::AddChainWithSubs:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::Add:
case RecurKind::FAdd:
case RecurKind::And:
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index f7bebf08c0d3b..6de2dab3a8608 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1097,6 +1097,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::Xor:
return Intrinsic::vector_reduce_xor;
case RecurKind::FMulAdd:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::FSub:
case RecurKind::FAdd:
return Intrinsic::vector_reduce_fadd;
@@ -1555,6 +1556,7 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::FMaximumNum:
return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
case RecurKind::FMulAdd:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::FSub:
case RecurKind::FAdd:
return Builder.CreateFAddReduce(getIdentity(), Src);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 22e4488200ef5..fe61c6bf10033 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29188,6 +29188,7 @@ class HorizontalReduction {
break;
case RecurKind::Sub:
case RecurKind::FSub:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -29340,6 +29341,7 @@ class HorizontalReduction {
return VectorizedValue;
case RecurKind::Sub:
case RecurKind::FSub:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -29444,6 +29446,7 @@ class HorizontalReduction {
}
case RecurKind::Sub:
case RecurKind::FSub:
+ case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
>From 525b91168e949ec05e663c594eba97505aee6a03 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 21 Apr 2026 15:41:58 +0000
Subject: [PATCH 11/19] rm untested FSub case in matchExtendedReductionOperand
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 10542570ab0e3..17ef3e26cbc61 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6128,8 +6128,7 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
// optimizeExtendsForPartialReduction.
Op = CastSource;
} else if (UpdateR->getOpcode() == Instruction::Add ||
- UpdateR->getOpcode() == Instruction::FAdd ||
- UpdateR->getOpcode() == Instruction::FSub) {
+ UpdateR->getOpcode() == Instruction::FAdd) {
// Match: UpdateR(PrevValue, ext(...))
// TODO: Remove the add/fadd restriction (we should be able to handle this
// case for sub reductions too).
>From 00db3094bf1b2ca9483805dcc4062955f1129380 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 27 Apr 2026 11:16:35 +0000
Subject: [PATCH 12/19] Add getSubRecurOpcode helper
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a4678fe6098b2..ba1c3f53e32db 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -556,6 +556,14 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
}
}
+Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
+ if (Kind == RecurKind::Sub)
+ return Instruction::Add;
+ if (Kind == RecurKind::FSub)
+ return Instruction::FAdd;
+ return (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(Kind);
+}
+
Value *VPInstruction::generate(VPTransformState &State) {
IRBuilderBase &Builder = State.Builder;
@@ -761,11 +769,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
else {
// For sub-recurrences, each part's reduction variable is already
// negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
- Instruction::BinaryOps Opcode =
- (RK == RecurKind::Sub) ? Instruction::Add
- : (RK == RecurKind::FSub)
- ? Instruction::FAdd
- : (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
+ Instruction::BinaryOps Opcode = getSubRecurOpcode(RK);
ReducedPartRdx =
Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
}
>From 397562cb264c49e2460b9675d39ccbbea16737a0 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 29 Apr 2026 13:04:03 +0000
Subject: [PATCH 13/19] Rm fallback in getSubRecurOpcode
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ba1c3f53e32db..7e76fb6c78317 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -561,7 +561,7 @@ Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
return Instruction::Add;
if (Kind == RecurKind::FSub)
return Instruction::FAdd;
- return (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(Kind);
+ llvm_unreachable("RecurKind should be Sub/FSub.");
}
Value *VPInstruction::generate(VPTransformState &State) {
@@ -769,7 +769,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
else {
// For sub-recurrences, each part's reduction variable is already
// negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
- Instruction::BinaryOps Opcode = getSubRecurOpcode(RK);
+ Instruction::BinaryOps Opcode =
+ RecurrenceDescriptor::isSubRecurrenceKind(RK)
+ ? getSubRecurOpcode(RK)
+ : (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
ReducedPartRdx =
Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
}
>From e452490d9a63d70d5b475da0b3b54c81c8310ea9 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 30 Apr 2026 12:22:32 +0000
Subject: [PATCH 14/19] Change fdotp_fsub test to do two fsub operations
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
.../AArch64/partial-reduce-sub.ll | 45 ++++++++++++++-----
2 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7e76fb6c78317..d76b53e64ed37 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -556,7 +556,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
}
}
-Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
+static Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
if (Kind == RecurKind::Sub)
return Instruction::Add;
if (Kind == RecurKind::FSub)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index f0363fe62d1ad..f7ed72f3a45e5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -269,9 +269,9 @@ exit:
ret i64 %sub
}
-define float @fdotp_fsub(ptr %a, ptr %b) #0 {
+define float @fdotp_fsub(ptr %a, ptr %b, ptr %c) #0 {
; CHECK-INTERLEAVE1-LABEL: define float @fdotp_fsub(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVE1-NEXT: entry:
; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-INTERLEAVE1: vector.ph:
@@ -283,10 +283,15 @@ define float @fdotp_fsub(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP8]], align 2
; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE1:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = fmul reassoc contract <8 x float> [[TMP9]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE1]], <8 x float> [[TMP10]])
; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -298,7 +303,7 @@ define float @fdotp_fsub(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVE1-NEXT: ret float [[TMP7]]
;
; CHECK-INTERLEAVED-LABEL: define float @fdotp_fsub(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVED-NEXT: entry:
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-INTERLEAVED: vector.ph:
@@ -315,14 +320,24 @@ define float @fdotp_fsub(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[TMP2]], i64 8
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x half>, ptr [[TMP2]], align 2
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = getelementptr half, ptr [[TMP16]], i64 8
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x half>, ptr [[TMP16]], align 2
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x half>, ptr [[TMP17]], align 2
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD3]] to <8 x float>
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fmul reassoc contract <8 x float> [[TMP4]], [[TMP5]]
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP6]])
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE1:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP6]])
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = fmul reassoc contract <8 x float> [[TMP7]], [[TMP8]]
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE5]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI1]], <8 x float> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI1]], <8 x float> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = fpext <8 x half> [[WIDE_LOAD5]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = fmul reassoc contract <8 x float> [[TMP18]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE1]], <8 x float> [[TMP13]])
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = fpext <8 x half> [[WIDE_LOAD6]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = fmul reassoc contract <8 x float> [[TMP14]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE5]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE7]], <8 x float> [[TMP15]])
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -335,7 +350,7 @@ define float @fdotp_fsub(ptr %a, ptr %b) #0 {
; CHECK-INTERLEAVED-NEXT: ret float [[TMP12]]
;
; CHECK-MAXBW-LABEL: define float @fdotp_fsub(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR0]] {
; CHECK-MAXBW-NEXT: entry:
; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK-MAXBW: vector.ph:
@@ -347,10 +362,15 @@ define float @fdotp_fsub(ptr %a, ptr %b) #0 {
; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP8]], align 2
; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE1:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = fmul reassoc contract <8 x float> [[TMP9]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE1]], <8 x float> [[TMP10]])
; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-MAXBW-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -373,8 +393,13 @@ for.body:
%gep.b = getelementptr half, ptr %b, i64 %iv
%load.b = load half, ptr %gep.b, align 2
%ext.b = fpext half %load.b to float
- %mul = fmul reassoc contract float %ext.b, %ext.a
- %sub = fsub reassoc contract float %accum, %mul
+ %gep.c = getelementptr half, ptr %c, i64 %iv
+ %load.c = load half, ptr %gep.c, align 2
+ %ext.c = fpext half %load.c to float
+ %mul.ab = fmul reassoc contract float %ext.b, %ext.a
+ %mul.ac = fmul reassoc contract float %ext.c, %ext.a
+ %sub.ab = fsub reassoc contract float %accum, %mul.ab
+ %sub = fsub reassoc contract float %sub.ab, %mul.ac
%iv.next = add i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %for.exit, label %for.body
>From fba8be2681f24ca54976c79772a5bf18b5f6bfd5 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 30 Apr 2026 14:55:34 +0000
Subject: [PATCH 15/19] pass fast-math flags
---
llvm/lib/Analysis/IVDescriptors.cpp | 9 ++++-----
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ++-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index c74cd9e0c7393..67954eaa663ee 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -93,7 +93,7 @@ static Instruction *lookThroughAnd(PHINode *Phi, Type *&RT,
}
bool RecurrenceDescriptor::isSubRecurrenceKind(RecurKind Kind) {
- return (Kind == RecurKind::Sub || Kind == RecurKind::FSub);
+ return Kind == RecurKind::Sub || Kind == RecurKind::FSub;
}
/// Compute the minimal bit width needed to represent a reduction whose exit
@@ -1019,10 +1019,9 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
Kind == RecurKind::FAddChainWithSubs,
I, I->hasAllowReassoc() ? nullptr : I);
case Instruction::Select:
- if (Kind == RecurKind::FAdd || Kind == RecurKind::FSub ||
+ if (isSubRecurrenceKind(Kind) || Kind == RecurKind::FAdd ||
Kind == RecurKind::FMul || Kind == RecurKind::Add ||
- Kind == RecurKind::Mul || Kind == RecurKind::Sub ||
- Kind == RecurKind::AddChainWithSubs ||
+ Kind == RecurKind::Mul || Kind == RecurKind::AddChainWithSubs ||
Kind == RecurKind::FAddChainWithSubs)
return isConditionalRdxPattern(I);
if (isFindRecurrenceKind(Kind) && SE)
@@ -1324,7 +1323,7 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
return true;
if (Cur->getOpcode() == Instruction::FSub &&
- Kind == RecurKind::AddChainWithSubs)
+ Kind == RecurKind::FAddChainWithSubs)
return true;
return Cur->getOpcode() == getOpcode();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c29de29ccf80c..ae727404501b7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8048,7 +8048,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
// For FP sub-reductions, verify start value is negative zero.
[[maybe_unused]] auto StartValueIsIdentity = [&] {
Value *IdentityValue = getRecurrenceIdentity(
- PhiR->getRecurrenceKind(), ResumeV->getType(), {});
+ PhiR->getRecurrenceKind(), ResumeV->getType(),
+ PhiR->getFastMathFlags());
auto *StartValue = dyn_cast<VPIRValue>(VPI->getOperand(0));
return StartValue && StartValue->getValue() == IdentityValue;
};
>From c041658ff03affa9c129826e2b93ccac557db51d Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 1 May 2026 13:24:59 +0000
Subject: [PATCH 16/19] add negative fadd_fsub test case
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +
.../partial-reduce-sub-epilogue-vec.ll | 203 ++++++++++++++++++
2 files changed, 205 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 17ef3e26cbc61..e02fae201b89e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5955,6 +5955,8 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// subtract in the middle block.
if (WidenRecipe->getOpcode() == Instruction::Sub &&
Chain.RK != RecurKind::Sub) {
+ assert((WidenRecipe->getOpcode() != Instruction::FSub) &&
+ "Fsub chain reduction isn't supported");
VPBuilder Builder(WidenRecipe);
Type *ElemTy = TypeInfo.inferScalarType(ExtendedOp);
auto *Zero = Plan.getZero(ElemTy);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index ce2af3b221306..cab6bd578f696 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -352,6 +352,209 @@ exit:
ret float %sub
}
+define float @fadd_fsub_reduction(float %startval, ptr %src1, ptr %src2, ptr %src3) #0 {
+; CHECK-EPI-LABEL: define float @fadd_fsub_reduction(
+; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]], ptr [[SRC3:%.*]]) #[[ATTR0]] {
+; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = insertelement <8 x float> splat (float -0.000000e+00), float [[STARTVAL]], i32 0
+; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-EPI: [[VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP1]], align 4
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x half>, ptr [[TMP2]], align 4
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP3]], align 4
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = fpext <16 x half> [[WIDE_LOAD]] to <16 x float>
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = fpext <16 x half> [[WIDE_LOAD1]] to <16 x float>
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = fmul reassoc contract <16 x float> [[TMP4]], [[TMP5]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI]], <16 x float> [[TMP6]])
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = fpext <16 x half> [[WIDE_LOAD2]] to <16 x float>
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = fmul reassoc contract <16 x float> [[TMP7]], [[TMP4]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[PARTIAL_REDUCE]], <16 x float> [[TMP8]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-EPI-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE3]])
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = insertelement <4 x float> splat (float -0.000000e+00), float [[BC_MERGE_RDX]], i32 0
+; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x half>, ptr [[TMP12]], align 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = fpext <4 x half> [[WIDE_LOAD6]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x half>, ptr [[TMP14]], align 4
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = fpext <4 x half> [[WIDE_LOAD7]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP16:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x half>, ptr [[TMP16]], align 4
+; CHECK-EPI-NEXT: [[TMP17:%.*]] = fpext <4 x half> [[WIDE_LOAD8]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP18:%.*]] = fmul reassoc contract <4 x float> [[TMP13]], [[TMP15]]
+; CHECK-EPI-NEXT: [[TMP19:%.*]] = fadd reassoc contract <4 x float> [[VEC_PHI5]], [[TMP18]]
+; CHECK-EPI-NEXT: [[TMP20:%.*]] = fmul reassoc contract <4 x float> [[TMP17]], [[TMP13]]
+; CHECK-EPI-NEXT: [[TMP21]] = fsub reassoc contract <4 x float> [[TMP19]], [[TMP20]]
+; CHECK-EPI-NEXT: [[INDEX_NEXT9]] = add nuw i32 [[INDEX4]], 4
+; CHECK-EPI-NEXT: [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT9]], 40
+; CHECK-EPI-NEXT: br i1 [[TMP22]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP23:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP21]])
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-EPI: [[LOOP]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-EPI-NEXT: [[SRC3_GEP:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC3_LOAD:%.*]] = load half, ptr [[SRC3_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC3_LOAD_EXT:%.*]] = fpext half [[SRC3_LOAD]] to float
+; CHECK-EPI-NEXT: [[MUL1:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-EPI-NEXT: [[ADD:%.*]] = fadd reassoc contract float [[ACCUM]], [[MUL1]]
+; CHECK-EPI-NEXT: [[MUL2:%.*]] = fmul reassoc contract float [[SRC3_LOAD_EXT]], [[SRC1_LOAD_EXT]]
+; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ADD]], [[MUL2]]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-EPI: [[EXIT]]:
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+; CHECK-PARTIAL-RED-EPI-LABEL: define float @fadd_fsub_reduction(
+; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]], ptr [[SRC3:%.*]]) #[[ATTR0]] {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = insertelement <8 x float> splat (float -0.000000e+00), float [[STARTVAL]], i32 0
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP1]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x half>, ptr [[TMP2]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP3]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fpext <16 x half> [[WIDE_LOAD]] to <16 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = fpext <16 x half> [[WIDE_LOAD1]] to <16 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = fmul reassoc contract <16 x float> [[TMP4]], [[TMP5]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI]], <16 x float> [[TMP6]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fpext <16 x half> [[WIDE_LOAD2]] to <16 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = fmul reassoc contract <16 x float> [[TMP7]], [[TMP4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[PARTIAL_REDUCE]], <16 x float> [[TMP8]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE3]])
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = insertelement <4 x float> splat (float -0.000000e+00), float [[BC_MERGE_RDX]], i32 0
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x half>, ptr [[TMP12]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x half>, ptr [[TMP13]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x half>, ptr [[TMP14]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fpext <8 x half> [[WIDE_LOAD6]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = fpext <8 x half> [[WIDE_LOAD7]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP17:%.*]] = fmul reassoc contract <8 x float> [[TMP15]], [[TMP16]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE9:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI5]], <8 x float> [[TMP17]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP18:%.*]] = fpext <8 x half> [[WIDE_LOAD8]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP19:%.*]] = fmul reassoc contract <8 x float> [[TMP18]], [[TMP15]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE10]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE9]], <8 x float> [[TMP19]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT11]] = add nuw i32 [[INDEX4]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT11]], 40
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP21:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE10]])
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX12]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_GEP:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_LOAD:%.*]] = load half, ptr [[SRC3_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_LOAD_EXT:%.*]] = fpext half [[SRC3_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL1:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ADD:%.*]] = fadd reassoc contract float [[ACCUM]], [[MUL1]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL2:%.*]] = fmul reassoc contract float [[SRC3_LOAD_EXT]], [[SRC1_LOAD_EXT]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ADD]], [[MUL2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi float [ %startval, %entry ], [ %sub, %loop ]
+ %src1.gep = getelementptr half, ptr %src1, i32 %iv
+ %src1.load = load half, ptr %src1.gep, align 4
+ %src1.load.ext = fpext half %src1.load to float
+ %src2.gep = getelementptr half, ptr %src2, i32 %iv
+ %src2.load = load half, ptr %src2.gep, align 4
+ %src2.load.ext = fpext half %src2.load to float
+ %src3.gep = getelementptr half, ptr %src3, i32 %iv
+ %src3.load = load half, ptr %src3.gep, align 4
+ %src3.load.ext = fpext half %src3.load to float
+ %mul1 = fmul reassoc contract float %src1.load.ext, %src2.load.ext
+ %add = fadd reassoc contract float %accum, %mul1
+ %mul2 = fmul reassoc contract float %src3.load.ext, %src1.load.ext
+ %sub = fsub reassoc contract float %add, %mul2
+ %iv.next = add i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv, 40
+ br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !0
+
+exit:
+ ret float %sub
+}
+
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 257edab92141aa1f02ae451882073ed886caed48 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 5 May 2026 11:05:44 +0000
Subject: [PATCH 17/19] Add nsz test and move fsub chain test to new file
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 5 +-
.../AArch64/partial-reduce-fsub-chained.ll | 36 +++
.../partial-reduce-sub-epilogue-vec.ll | 294 +++++++-----------
3 files changed, 158 insertions(+), 177 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e02fae201b89e..e73c851df7cdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5955,8 +5955,6 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// subtract in the middle block.
if (WidenRecipe->getOpcode() == Instruction::Sub &&
Chain.RK != RecurKind::Sub) {
- assert((WidenRecipe->getOpcode() != Instruction::FSub) &&
- "Fsub chain reduction isn't supported");
VPBuilder Builder(WidenRecipe);
Type *ElemTy = TypeInfo.inferScalarType(ExtendedOp);
auto *Zero = Plan.getZero(ElemTy);
@@ -5967,6 +5965,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
ExtendedOp = NegRecipe;
}
+ assert((Chain.RK != RecurKind::FAddChainWithSubs) &&
+ "FSub chain reduction isn't supported");
+
// FIXME: Do these transforms before invoking the cost-model.
ExtendedOp = optimizeExtendsForPartialReduction(ExtendedOp, TypeInfo);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
new file mode 100644
index 0000000000000..eb6657f2ad295
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; REQUIRES: asserts
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s
+; RUN: not --crash opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define float @fadd_fsub_reduction(float %startval, ptr %src1, ptr %src2, ptr %src3) #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi float [ %startval, %entry ], [ %sub, %loop ]
+ %src1.gep = getelementptr half, ptr %src1, i32 %iv
+ %src1.load = load half, ptr %src1.gep, align 4
+ %src1.load.ext = fpext half %src1.load to float
+ %src2.gep = getelementptr half, ptr %src2, i32 %iv
+ %src2.load = load half, ptr %src2.gep, align 4
+ %src2.load.ext = fpext half %src2.load to float
+ %src3.gep = getelementptr half, ptr %src3, i32 %iv
+ %src3.load = load half, ptr %src3.gep, align 4
+ %src3.load.ext = fpext half %src3.load to float
+ %mul1 = fmul reassoc contract float %src1.load.ext, %src2.load.ext
+ %add = fadd reassoc contract float %accum, %mul1
+ %mul2 = fmul reassoc contract float %src3.load.ext, %src1.load.ext
+ %sub = fsub reassoc contract float %add, %mul2
+ %iv.next = add i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv, 1024
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret float %sub
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index cab6bd578f696..21c9b9cde13ba 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -352,209 +352,153 @@ exit:
ret float %sub
}
-define float @fadd_fsub_reduction(float %startval, ptr %src1, ptr %src2, ptr %src3) #0 {
-; CHECK-EPI-LABEL: define float @fadd_fsub_reduction(
-; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]], ptr [[SRC3:%.*]]) #[[ATTR0]] {
+define float @fsub_reduction_nsz(ptr %a, ptr %b, ptr %c, i64 %n) {
+; CHECK-EPI-LABEL: define float @fsub_reduction_nsz(
+; CHECK-EPI-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-EPI: [[VECTOR_PH]]:
-; CHECK-EPI-NEXT: [[TMP0:%.*]] = insertelement <8 x float> splat (float -0.000000e+00), float [[STARTVAL]], i32 0
; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-EPI: [[VECTOR_BODY]]:
-; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP1]], align 4
-; CHECK-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x half>, ptr [[TMP2]], align 4
-; CHECK-EPI-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP3]], align 4
-; CHECK-EPI-NEXT: [[TMP4:%.*]] = fpext <16 x half> [[WIDE_LOAD]] to <16 x float>
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = fpext <16 x half> [[WIDE_LOAD1]] to <16 x float>
-; CHECK-EPI-NEXT: [[TMP6:%.*]] = fmul reassoc contract <16 x float> [[TMP4]], [[TMP5]]
-; CHECK-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI]], <16 x float> [[TMP6]])
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = fpext <16 x half> [[WIDE_LOAD2]] to <16 x float>
-; CHECK-EPI-NEXT: [[TMP8:%.*]] = fmul reassoc contract <16 x float> [[TMP7]], [[TMP4]]
-; CHECK-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[PARTIAL_REDUCE]], <16 x float> [[TMP8]])
-; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-EPI-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
-; CHECK-EPI-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP2]], align 2
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract <8 x float> [[TMP3]], [[TMP4]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc nsz contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP5]])
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = fmul reassoc nsz contract <8 x float> [[TMP6]], [[TMP4]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc nsz contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE]], <8 x float> [[TMP7]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-EPI: [[MIDDLE_BLOCK]]:
-; CHECK-EPI-NEXT: [[TMP10:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE3]])
-; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = call reassoc nsz contract float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PARTIAL_REDUCE3]])
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = fsub float 0.000000e+00, [[TMP9]]
+; CHECK-EPI-NEXT: br i1 true, label %[[FOR_EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF10:![0-9]+]]
; CHECK-EPI: [[VEC_EPILOG_PH]]:
-; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[TMP11:%.*]] = insertelement <4 x float> splat (float -0.000000e+00), float [[BC_MERGE_RDX]], i32 0
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 1024, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-EPI-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP21:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX4]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x half>, ptr [[TMP12]], align 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = fpext <4 x half> [[WIDE_LOAD6]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP14:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX4]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x half>, ptr [[TMP14]], align 4
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = fpext <4 x half> [[WIDE_LOAD7]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP16:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX4]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x half>, ptr [[TMP16]], align 4
+; CHECK-EPI-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <2 x float> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x half>, ptr [[TMP11]], align 2
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x half>, ptr [[TMP12]], align 2
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x half>, ptr [[TMP13]], align 2
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = fpext <4 x half> [[WIDE_LOAD7]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = fpext <4 x half> [[WIDE_LOAD6]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP16:%.*]] = fmul reassoc nsz contract <4 x float> [[TMP14]], [[TMP15]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE9:%.*]] = call reassoc nsz contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI5]], <4 x float> [[TMP16]])
; CHECK-EPI-NEXT: [[TMP17:%.*]] = fpext <4 x half> [[WIDE_LOAD8]] to <4 x float>
-; CHECK-EPI-NEXT: [[TMP18:%.*]] = fmul reassoc contract <4 x float> [[TMP13]], [[TMP15]]
-; CHECK-EPI-NEXT: [[TMP19:%.*]] = fadd reassoc contract <4 x float> [[VEC_PHI5]], [[TMP18]]
-; CHECK-EPI-NEXT: [[TMP20:%.*]] = fmul reassoc contract <4 x float> [[TMP17]], [[TMP13]]
-; CHECK-EPI-NEXT: [[TMP21]] = fsub reassoc contract <4 x float> [[TMP19]], [[TMP20]]
-; CHECK-EPI-NEXT: [[INDEX_NEXT9]] = add nuw i32 [[INDEX4]], 4
-; CHECK-EPI-NEXT: [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT9]], 40
-; CHECK-EPI-NEXT: br i1 [[TMP22]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-EPI-NEXT: [[TMP18:%.*]] = fmul reassoc nsz contract <4 x float> [[TMP17]], [[TMP15]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE10]] = call reassoc nsz contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[PARTIAL_REDUCE9]], <4 x float> [[TMP18]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX4]], 4
+; CHECK-EPI-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT11]], 1024
+; CHECK-EPI-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-EPI-NEXT: [[TMP23:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP21]])
-; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI-NEXT: [[TMP20:%.*]] = call reassoc nsz contract float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[PARTIAL_REDUCE10]])
+; CHECK-EPI-NEXT: [[TMP21:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP20]]
+; CHECK-EPI-NEXT: br i1 true, label %[[FOR_EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
-; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
-; CHECK-EPI: [[LOOP]]:
-; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
-; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
-; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
-; CHECK-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
-; CHECK-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
-; CHECK-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
-; CHECK-EPI-NEXT: [[SRC3_GEP:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[IV]]
-; CHECK-EPI-NEXT: [[SRC3_LOAD:%.*]] = load half, ptr [[SRC3_GEP]], align 4
-; CHECK-EPI-NEXT: [[SRC3_LOAD_EXT:%.*]] = fpext half [[SRC3_LOAD]] to float
-; CHECK-EPI-NEXT: [[MUL1:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
-; CHECK-EPI-NEXT: [[ADD:%.*]] = fadd reassoc contract float [[ACCUM]], [[MUL1]]
-; CHECK-EPI-NEXT: [[MUL2:%.*]] = fmul reassoc contract float [[SRC3_LOAD_EXT]], [[SRC1_LOAD_EXT]]
-; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ADD]], [[MUL2]]
-; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
-; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK-EPI: [[EXIT]]:
-; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1024, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-EPI: [[FOR_BODY]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX12]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[FOR_BODY]] ]
+; CHECK-EPI-NEXT: [[GEP_A:%.*]] = getelementptr half, ptr [[A]], i64 [[IV]]
+; CHECK-EPI-NEXT: [[LOAD_A:%.*]] = load half, ptr [[GEP_A]], align 2
+; CHECK-EPI-NEXT: [[EXT_A:%.*]] = fpext half [[LOAD_A]] to float
+; CHECK-EPI-NEXT: [[GEP_B:%.*]] = getelementptr half, ptr [[B]], i64 [[IV]]
+; CHECK-EPI-NEXT: [[LOAD_B:%.*]] = load half, ptr [[GEP_B]], align 2
+; CHECK-EPI-NEXT: [[EXT_B:%.*]] = fpext half [[LOAD_B]] to float
+; CHECK-EPI-NEXT: [[GEP_C:%.*]] = getelementptr half, ptr [[C]], i64 [[IV]]
+; CHECK-EPI-NEXT: [[LOAD_C:%.*]] = load half, ptr [[GEP_C]], align 2
+; CHECK-EPI-NEXT: [[EXT_C:%.*]] = fpext half [[LOAD_C]] to float
+; CHECK-EPI-NEXT: [[MUL_AB:%.*]] = fmul reassoc nsz contract float [[EXT_B]], [[EXT_A]]
+; CHECK-EPI-NEXT: [[MUL_AC:%.*]] = fmul reassoc nsz contract float [[EXT_C]], [[EXT_A]]
+; CHECK-EPI-NEXT: [[SUB_AB:%.*]] = fsub reassoc nsz contract float [[ACCUM]], [[MUL_AB]]
+; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc nsz contract float [[SUB_AB]], [[MUL_AC]]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-EPI: [[FOR_EXIT]]:
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
;
-; CHECK-PARTIAL-RED-EPI-LABEL: define float @fadd_fsub_reduction(
-; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]], ptr [[SRC3:%.*]]) #[[ATTR0]] {
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
-; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI-LABEL: define float @fsub_reduction_nsz(
+; CHECK-PARTIAL-RED-EPI-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ENTRY:.*:]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = insertelement <8 x float> splat (float -0.000000e+00), float [[STARTVAL]], i32 0
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP1]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x half>, ptr [[TMP2]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP3]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fpext <16 x half> [[WIDE_LOAD]] to <16 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = fpext <16 x half> [[WIDE_LOAD1]] to <16 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = fmul reassoc contract <16 x float> [[TMP4]], [[TMP5]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[VEC_PHI]], <16 x float> [[TMP6]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fpext <16 x half> [[WIDE_LOAD2]] to <16 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = fmul reassoc contract <16 x float> [[TMP7]], [[TMP4]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> [[PARTIAL_REDUCE]], <16 x float> [[TMP8]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP2]], align 2
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract <8 x float> [[TMP3]], [[TMP4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc nsz contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP5]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fmul reassoc nsz contract <8 x float> [[TMP6]], [[TMP4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE3]] = call reassoc nsz contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE]], <8 x float> [[TMP7]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[PARTIAL_REDUCE3]])
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = insertelement <4 x float> splat (float -0.000000e+00), float [[BC_MERGE_RDX]], i32 0
-; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX4:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x float> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX4]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x half>, ptr [[TMP12]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX4]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x half>, ptr [[TMP13]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[INDEX4]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x half>, ptr [[TMP14]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fpext <8 x half> [[WIDE_LOAD6]] to <8 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = fpext <8 x half> [[WIDE_LOAD7]] to <8 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP17:%.*]] = fmul reassoc contract <8 x float> [[TMP15]], [[TMP16]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE9:%.*]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI5]], <8 x float> [[TMP17]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP18:%.*]] = fpext <8 x half> [[WIDE_LOAD8]] to <8 x float>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP19:%.*]] = fmul reassoc contract <8 x float> [[TMP18]], [[TMP15]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE10]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[PARTIAL_REDUCE9]], <8 x float> [[TMP19]])
-; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT11]] = add nuw i32 [[INDEX4]], 8
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT11]], 40
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP21:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE10]])
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
-; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 40, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX12:%.*]] = phi float [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
-; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX12]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_GEP:%.*]] = getelementptr half, ptr [[SRC3]], i32 [[IV]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_LOAD:%.*]] = load half, ptr [[SRC3_GEP]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC3_LOAD_EXT:%.*]] = fpext half [[SRC3_LOAD]] to float
-; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL1:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ADD:%.*]] = fadd reassoc contract float [[ACCUM]], [[MUL1]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL2:%.*]] = fmul reassoc contract float [[SRC3_LOAD_EXT]], [[SRC1_LOAD_EXT]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ADD]], [[MUL2]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 40
-; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
-; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = call reassoc nsz contract float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PARTIAL_REDUCE3]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = fsub float 0.000000e+00, [[TMP9]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[FOR_EXIT:.*]]
+; CHECK-PARTIAL-RED-EPI: [[FOR_EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[TMP10]]
;
entry:
- br label %loop
+ br label %for.body
-loop:
- %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
- %accum = phi float [ %startval, %entry ], [ %sub, %loop ]
- %src1.gep = getelementptr half, ptr %src1, i32 %iv
- %src1.load = load half, ptr %src1.gep, align 4
- %src1.load.ext = fpext half %src1.load to float
- %src2.gep = getelementptr half, ptr %src2, i32 %iv
- %src2.load = load half, ptr %src2.gep, align 4
- %src2.load.ext = fpext half %src2.load to float
- %src3.gep = getelementptr half, ptr %src3, i32 %iv
- %src3.load = load half, ptr %src3.gep, align 4
- %src3.load.ext = fpext half %src3.load to float
- %mul1 = fmul reassoc contract float %src1.load.ext, %src2.load.ext
- %add = fadd reassoc contract float %accum, %mul1
- %mul2 = fmul reassoc contract float %src3.load.ext, %src1.load.ext
- %sub = fsub reassoc contract float %add, %mul2
- %iv.next = add i32 %iv, 1
- %exitcond.not = icmp eq i32 %iv, 40
- br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !0
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi float [ 0.0, %entry ], [ %sub, %for.body ]
+ %gep.a = getelementptr half, ptr %a, i64 %iv
+ %load.a = load half, ptr %gep.a, align 2
+ %ext.a = fpext half %load.a to float
+ %gep.b = getelementptr half, ptr %b, i64 %iv
+ %load.b = load half, ptr %gep.b, align 2
+ %ext.b = fpext half %load.b to float
+ %gep.c = getelementptr half, ptr %c, i64 %iv
+ %load.c = load half, ptr %gep.c, align 2
+ %ext.c = fpext half %load.c to float
+ %mul.ab = fmul nsz reassoc contract float %ext.b, %ext.a
+ %mul.ac = fmul nsz reassoc contract float %ext.c, %ext.a
+ %sub.ab = fsub nsz reassoc contract float %accum, %mul.ab
+ %sub = fsub nsz reassoc contract float %sub.ab, %mul.ac
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %for.exit, label %for.body
-exit:
+for.exit:
ret float %sub
}
-
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
!0 = distinct !{!0, !1}
>From 0263f4bca65277aca5b74ff93947c94883031e93 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 5 May 2026 12:55:51 +0000
Subject: [PATCH 18/19] Add FileCheck for assertion
---
.../LoopVectorize/AArch64/partial-reduce-fsub-chained.ll | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
index eb6657f2ad295..56691914c8b12 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
@@ -1,12 +1,12 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; REQUIRES: asserts
-; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s
-; RUN: not --crash opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s
-; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
+; RUN: not --crash opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"
+; ASSERTION: Assertion `(Chain.RK != RecurKind::FAddChainWithSubs)
define float @fadd_fsub_reduction(float %startval, ptr %src1, ptr %src2, ptr %src3) #0 {
entry:
br label %loop
>From 783ecf118c7525ac727a82e5465e66411fd74dc8 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 6 May 2026 10:43:42 +0000
Subject: [PATCH 19/19] Change assertion string to pass FileCheck on all builds
---
.../LoopVectorize/AArch64/partial-reduce-fsub-chained.ll | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
index 56691914c8b12..8a617dfe42925 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fsub-chained.ll
@@ -1,12 +1,12 @@
; REQUIRES: asserts
-; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
-; RUN: not --crash opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
-; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S 2>&1 %s | FileCheck %s --check-prefix=ASSERTION
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S %s 2>&1 | FileCheck %s --check-prefix=ASSERTION
+; RUN: not --crash opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S %s 2>&1 | FileCheck %s --check-prefix=ASSERTION
+; RUN: not --crash opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S %s 2>&1 | FileCheck %s --check-prefix=ASSERTION
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"
-; ASSERTION: Assertion `(Chain.RK != RecurKind::FAddChainWithSubs)
+; ASSERTION: (Chain.RK != RecurKind::FAddChainWithSubs)
define float @fadd_fsub_reduction(float %startval, ptr %src1, ptr %src2, ptr %src3) #0 {
entry:
br label %loop
More information about the llvm-commits
mailing list