[llvm] [LV] Handle FSub Partial Reductions (PR #191186)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 04:05:11 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/191186
>From 21e83dfeffaab007c3564b85899816b2a636402b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 30 Mar 2026 15:28:54 +0000
Subject: [PATCH 1/2] [LV] Handle FSub Partial Reductions
Introduces a new RecurKind value 'FSub' in order to handle partial
reductions of floating point values.
This is done by following the existing method for integer partial
reductions, doing a positive accumulation followed by a final
subtraction in the middle block.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 1 +
llvm/lib/Analysis/IVDescriptors.cpp | 13 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 4 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 2 +
.../Transforms/Vectorize/LoopVectorize.cpp | 27 ++-
.../Transforms/Vectorize/VPlanPatternMatch.h | 30 ++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 +-
.../partial-reduce-sub-epilogue-vec.ll | 169 ++++++++++++++++++
.../AArch64/partial-reduce-sub.ll | 114 ++++++++++++
10 files changed, 356 insertions(+), 17 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 8383da0f9e879..8cfb8a78a9a96 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -46,6 +46,7 @@ enum class RecurKind {
UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
FAdd, ///< Sum of floats.
+ FSub, ///< Subtraction of floats.
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 185943e0a9d2e..4ebbe605c0603 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1009,13 +1009,16 @@ RecurrenceDescriptor::isRecurrenceInstr(Loop *L, PHINode *OrigPhi,
return InstDesc(Kind == RecurKind::FMul, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::FSub:
+ return InstDesc(Kind == RecurKind::FSub, I,
+ I->hasAllowReassoc() ? nullptr : I);
case Instruction::FAdd:
return InstDesc(Kind == RecurKind::FAdd, I,
I->hasAllowReassoc() ? nullptr : I);
case Instruction::Select:
- if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
- Kind == RecurKind::Add || Kind == RecurKind::Mul ||
- Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs)
+ if (Kind == RecurKind::FAdd || Kind == RecurKind::FSub ||
+ Kind == RecurKind::FMul || Kind == RecurKind::Add ||
+ Kind == RecurKind::Mul || Kind == RecurKind::Sub ||
+ Kind == RecurKind::AddChainWithSubs)
return isConditionalRdxPattern(I);
if (isFindRecurrenceKind(Kind) && SE)
return isFindPattern(L, OrigPhi, I, *SE);
@@ -1104,6 +1107,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
return true;
}
+ if (AddReductionVar(Phi, RecurKind::FSub, TheLoop, RedDes, DB, AC, DT, SE)) {
+ LLVM_DEBUG(dbgs() << "Found an FSub reduction PHI." << *Phi << "\n");
+ return true;
+ }
if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, RedDes, DB, AC, DT, SE)) {
LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
return true;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 734339e5c7a05..eff7deb76e2ad 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5930,7 +5930,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
return Invalid;
if ((Opcode != Instruction::Add && Opcode != Instruction::Sub &&
- Opcode != Instruction::FAdd) ||
+ Opcode != Instruction::FAdd && Opcode != Instruction::FSub) ||
OpAExtend == TTI::PR_None)
return Invalid;
@@ -5997,7 +5997,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
NEONPred);
};
- bool IsSub = Opcode == Instruction::Sub;
+ bool IsSub = (Opcode == Instruction::Sub) || (Opcode == Instruction::FSub);
InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
if (AccumLT.second.getScalarType() == MVT::i32 &&
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 7573467917c73..f4c69e76a1e9f 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1097,6 +1097,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::Xor:
return Intrinsic::vector_reduce_xor;
case RecurKind::FMulAdd:
+ case RecurKind::FSub:
case RecurKind::FAdd:
return Intrinsic::vector_reduce_fadd;
case RecurKind::FMul:
@@ -1493,6 +1494,7 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::FMaximumNum:
return Builder.CreateUnaryIntrinsic(getReductionIntrinsicID(RdxKind), Src);
case RecurKind::FMulAdd:
+ case RecurKind::FSub:
case RecurKind::FAdd:
return Builder.CreateFAddReduce(getIdentity(), Src);
case RecurKind::FMul:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1fc880208ebf7..1954abeadb3fc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9058,17 +9058,30 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
// Partial sub-reductions always start at 0 and account for the
// reduction start value in a final subtraction. Update it to use the
// resume value from the main vector loop.
- if (PhiR->getVFScaleFactor() > 1 &&
- PhiR->getRecurrenceKind() == RecurKind::Sub) {
+ if ((PhiR->getVFScaleFactor() > 1 &&
+ PhiR->getRecurrenceKind() == RecurKind::Sub) ||
+ PhiR->getRecurrenceKind() == RecurKind::FSub) {
auto *Sub = cast<VPInstruction>(RdxResult->getSingleUser());
- assert(Sub->getOpcode() == Instruction::Sub && "Unexpected opcode");
+ assert((Sub->getOpcode() == Instruction::Sub ||
+ Sub->getOpcode() == Instruction::FSub) &&
+ "Unexpected opcode");
assert(isa<VPIRValue>(Sub->getOperand(0)) &&
"Expected operand to match the original start value of the "
"reduction");
- assert(VPlanPatternMatch::match(VPI->getOperand(0),
- VPlanPatternMatch::m_ZeroInt()) &&
- "Expected start value for partial sub-reduction to start at "
- "zero");
+ // For integer sub-reductions, verify start value is zero.
+ // For FP sub-reductions, verify start value is negative zero.
+ if (PhiR->getRecurrenceKind() == RecurKind::Sub) {
+ assert(VPlanPatternMatch::match(VPI->getOperand(0),
+ VPlanPatternMatch::m_ZeroInt()) &&
+ "Expected start value for partial sub-reduction to start "
+ "at zero");
+ } else if (PhiR->getRecurrenceKind() == RecurKind::FSub) {
+ assert(
+ VPlanPatternMatch::match(VPI->getOperand(0),
+ VPlanPatternMatch::m_NegZeroFP()) &&
+ "Expected start value for partial sub-reduction to start "
+ "at negative zero");
+ }
Sub->setOperand(0, StartVal);
} else
VPI->setOperand(0, StartVal);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 2ce012c5a9f10..df5dc88c25b06 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -184,6 +184,36 @@ inline int_pred_ty<is_zero_int> m_ZeroInt() {
/// For vectors, this includes constants with undefined elements.
inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
+/// Match a floating-point constant if Pred::isValue returns true for the
+/// APFloat.
+template <typename Pred> struct fp_pred_ty {
+ Pred P;
+
+ fp_pred_ty(Pred P) : P(std::move(P)) {}
+ fp_pred_ty() : P() {}
+
+ bool match(const VPValue *VPV) const {
+ auto *VPI = dyn_cast<VPInstruction>(VPV);
+ if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
+ VPV = VPI->getOperand(0);
+ auto *VPIRV = dyn_cast<VPIRValue>(VPV);
+ if (!VPIRV)
+ return false;
+ auto *CFP = dyn_cast<ConstantFP>(VPIRV->getValue());
+ if (!CFP)
+ return false;
+ return P.isValue(CFP->getValueAPF());
+ }
+};
+struct is_neg_zero_fp {
+ bool isValue(const APFloat &C) const { return C.isNegZero(); }
+};
+
+/// Match a floating-point negative zero.
+inline fp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
+ return fp_pred_ty<is_neg_zero_fp>();
+}
+
struct bind_apint {
const APInt *&Res;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc442c22d1256..bece327a5ba29 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -766,8 +766,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
// For sub-recurrences, each part's reduction variable is already
// negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
Instruction::BinaryOps Opcode =
- RK == RecurKind::Sub
- ? Instruction::Add
+ (RK == RecurKind::Sub) ? Instruction::Add
+ : (RK == RecurKind::FSub)
+ ? Instruction::FAdd
: (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
ReducedPartRdx =
Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4264a216e2455..af2db5a3246b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6135,7 +6135,7 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// If this is the last value in a sub-reduction chain, then update the PHI
// node to start at `0` and update the reduction-result to subtract from
// the PHI's start value.
- if (Chain.RK != RecurKind::Sub)
+ if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
return;
VPValue *OldStartValue = StartInst->getOperand(0);
@@ -6146,7 +6146,8 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
assert(RdxResult && "Could not find reduction result");
VPBuilder Builder = VPBuilder::getToInsertAfter(RdxResult);
- constexpr unsigned SubOpc = Instruction::BinaryOps::Sub;
+ unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
+ : Instruction::BinaryOps::Sub;
VPInstruction *NewResult = Builder.createNaryOp(
SubOpc, {OldStartValue, RdxResult}, VPIRFlags::getDefaultFlags(SubOpc),
RdxPhi->getDebugLoc());
@@ -6248,7 +6249,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
Op = CastSource;
OuterExtKind = getPartialReductionExtendKind(CastRecipe);
} else if (UpdateR->getOpcode() == Instruction::Add ||
- UpdateR->getOpcode() == Instruction::FAdd) {
+ UpdateR->getOpcode() == Instruction::FAdd ||
+ UpdateR->getOpcode() == Instruction::FSub) {
// Match: UpdateR(PrevValue, ext(...))
// TODO: Remove the add/fadd restriction (we should be able to handle this
// case for sub reductions too).
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index 8db3a3294d7bc..b210f8b49f2e3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -185,4 +185,173 @@ exit:
ret i32 %sub
}
+define float @fsub_reduction(float %startval, ptr %src1, ptr %src2) #0 {
+; CHECK-EPI-LABEL: define float @fsub_reduction(
+; CHECK-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
+; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-EPI: [[VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
+; CHECK-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x half>, ptr [[TMP8]], align 4
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x half>, ptr [[TMP9]], align 4
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = fpext <4 x half> [[WIDE_LOAD4]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = fpext <4 x half> [[WIDE_LOAD5]] to <4 x float>
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <4 x float> [[TMP10]], [[TMP11]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI3]], <4 x float> [[TMP12]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 36
+; CHECK-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE6]])
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-EPI: [[LOOP]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-EPI: [[EXIT]]:
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+; CHECK-PARTIAL-RED-EPI-LABEL: define float @fsub_reduction(
+; CHECK-PARTIAL-RED-EPI-SAME: float [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0]] {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ splat (float -0.000000e+00), %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x half>, ptr [[TMP1]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = fpext <4 x half> [[WIDE_LOAD]] to <4 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = fpext <4 x half> [[WIDE_LOAD1]] to <4 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = fmul reassoc contract <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> [[VEC_PHI]], <4 x float> [[TMP4]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 36
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> [[PARTIAL_REDUCE]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = fsub float [[STARTVAL]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP8]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[INDEX2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x half>, ptr [[TMP9]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = fpext <8 x half> [[WIDE_LOAD5]] to <8 x float>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = fmul reassoc contract <8 x float> [[TMP10]], [[TMP11]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI3]], <8 x float> [[TMP12]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE6]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = fsub float [[BC_MERGE_RDX]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 36, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr half, ptr [[SRC1]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load half, ptr [[SRC1_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = fpext half [[SRC1_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr half, ptr [[SRC2]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD:%.*]] = load half, ptr [[SRC2_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = fpext half [[SRC2_LOAD]] to float
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL:%.*]] = fmul reassoc contract float [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = fsub reassoc contract float [[ACCUM]], [[MUL]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi float [ [[SUB]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: ret float [[SUB_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi float [ %startval, %entry ], [ %sub, %loop ]
+ %src1.gep = getelementptr half, ptr %src1, i32 %iv
+ %src1.load = load half, ptr %src1.gep, align 4
+ %src1.load.ext = fpext half %src1.load to float
+ %src2.gep = getelementptr half, ptr %src2, i32 %iv
+ %src2.load = load half, ptr %src2.gep, align 4
+ %src2.load.ext = fpext half %src2.load to float
+ %mul = fmul reassoc contract float %src1.load.ext, %src2.load.ext
+ %sub = fsub reassoc contract float %accum, %mul
+ %iv.next = add i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv, 38
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret float %sub
+}
+
+
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index 9bf37a43e467d..91cfe948bafdf 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -280,6 +280,120 @@ exit:
ret i64 %sub
}
+define float @fdotp_fsub(ptr %a, ptr %b) #0 {
+; CHECK-INTERLEAVE1-LABEL: define float @fdotp_fsub(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: entry:
+; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1: vector.ph:
+; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1: vector.body:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1: middle.block:
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVE1: for.exit:
+; CHECK-INTERLEAVE1-NEXT: ret float [[TMP7]]
+;
+; CHECK-INTERLEAVED-LABEL: define float @fdotp_fsub(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: entry:
+; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED: vector.ph:
+; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED: vector.body:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[TMP0]], i64 8
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[TMP2]], i64 8
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x half>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD3]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fmul reassoc contract <8 x float> [[TMP4]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP6]])
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = fpext <8 x half> [[WIDE_LOAD2]] to <8 x float>
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = fmul reassoc contract <8 x float> [[TMP7]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE5]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI1]], <8 x float> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED: middle.block:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <4 x float> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = fsub float 0.000000e+00, [[TMP11]]
+; CHECK-INTERLEAVED-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVED: for.exit:
+; CHECK-INTERLEAVED-NEXT: ret float [[TMP12]]
+;
+; CHECK-MAXBW-LABEL: define float @fdotp_fsub(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: entry:
+; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW: vector.ph:
+; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW: vector.body:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 2
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x half>, ptr [[TMP1]], align 2
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = fpext <8 x half> [[WIDE_LOAD1]] to <8 x float>
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float>
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = fmul reassoc contract <8 x float> [[TMP2]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-MAXBW-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW: middle.block:
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]]
+; CHECK-MAXBW-NEXT: br label [[FOR_EXIT:%.*]]
+; CHECK-MAXBW: for.exit:
+; CHECK-MAXBW-NEXT: ret float [[TMP7]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi float [ 0.0, %entry ], [ %sub, %for.body ]
+ %gep.a = getelementptr half, ptr %a, i64 %iv
+ %load.a = load half, ptr %gep.a, align 2
+ %ext.a = fpext half %load.a to float
+ %gep.b = getelementptr half, ptr %b, i64 %iv
+ %load.b = load half, ptr %gep.b, align 2
+ %ext.b = fpext half %load.b to float
+ %mul = fmul reassoc contract float %ext.b, %ext.a
+ %sub = fsub reassoc contract float %accum, %mul
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:
+ ret float %sub
+}
+
!7 = distinct !{!7, !8, !9, !10}
!8 = !{!"llvm.loop.mustprogress"}
!9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
>From 13f5fc20459abfa5dc6da6afadcd60e9625b99b0 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 14 Apr 2026 09:48:41 +0000
Subject: [PATCH 2/2] Add missing RecurKind::FSub cases in SLPVectorizer
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 16ef024561e26..624087898b3ee 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28227,6 +28227,7 @@ class HorizontalReduction {
// res = vv
break;
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -28378,6 +28379,7 @@ class HorizontalReduction {
// res = vv
return VectorizedValue;
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
@@ -28481,6 +28483,7 @@ class HorizontalReduction {
return Builder.CreateFMul(VectorizedValue, Scale);
}
case RecurKind::Sub:
+ case RecurKind::FSub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
More information about the llvm-commits
mailing list