[llvm] [LLVM] Add `llvm.vector.reduce.fdot` intrinsic (PR #192174)
Princeton Ferro via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 03:56:04 PDT 2026
https://github.com/Prince781 updated https://github.com/llvm/llvm-project/pull/192174
>From 182c2d65442557ee95dbdd1db51dffe0f14d8f0c Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Wed, 8 Apr 2026 13:14:54 -0700
Subject: [PATCH 01/17] Add llvm.vector.reduce.fdot intrinsic and
VECREDUCE_FDOT ISD node
Introduces a new floating-point dot product reduction intrinsic
`llvm.vector.reduce.fdot(acc, vecA, vecB)` that computes
`acc + sum(vecA[i] * vecB[i])`. Semantics are controlled by fast-math
flags: default uses sequential fmul+fadd pairs, `contract` enables an
FMA chain, and `reassoc` permits reordering. Adds the corresponding
VECREDUCE_FDOT SelectionDAG node with full type legalization support
(scalarize, split, widen, soften), IR-level expansion fallback, GlobalISel
lowering via G_VECREDUCE_SEQ_FADD/G_VECREDUCE_FADD, IR verifier checks,
and codegen tests for AArch64, AMDGPU, NVPTX, Thumb2/MVE, X86, and the
generic expand path.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/docs/LangRef.rst | 56 +++++++
llvm/include/llvm/CodeGen/ISDOpcodes.h | 7 +
llvm/include/llvm/CodeGen/TargetLowering.h | 5 +
llvm/include/llvm/IR/Intrinsics.td | 4 +
llvm/lib/CodeGen/ExpandReductions.cpp | 26 +++
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 21 +++
.../SelectionDAG/LegalizeFloatTypes.cpp | 15 ++
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 5 +
.../SelectionDAG/LegalizeVectorOps.cpp | 4 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 78 +++++++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 9 +
.../SelectionDAG/SelectionDAGDumper.cpp | 1 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 37 +++++
llvm/lib/CodeGen/TargetLoweringBase.cpp | 3 +-
llvm/lib/IR/Verifier.cpp | 10 ++
llvm/test/CodeGen/AArch64/vecreduce-fdot.ll | 156 ++++++++++++++++++
.../test/CodeGen/AMDGPU/vector-reduce-fdot.ll | 36 ++++
.../CodeGen/Generic/expand-fdot-reduction.ll | 51 ++++++
llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll | 82 +++++++++
.../test/CodeGen/Thumb2/mve-vecreduce-fdot.ll | 31 ++++
llvm/test/CodeGen/X86/vector-reduce-fdot.ll | 97 +++++++++++
21 files changed, 733 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
create mode 100644 llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
create mode 100644 llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
create mode 100644 llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
create mode 100644 llvm/test/CodeGen/X86/vector-reduce-fdot.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 6f34005f3e945..aa2cf87ecbf86 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -20784,6 +20784,62 @@ Examples:
%unord = call reassoc float @llvm.vector.reduce.fmul.v4f32(float 1.0, <4 x float> %input) ; relaxed reduction
%ord = call float @llvm.vector.reduce.fmul.v4f32(float %start_value, <4 x float> %input) ; sequential reduction
+.. _int_vector_reduce_fdot:
+
+'``llvm.vector.reduce.fdot.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+ declare float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ declare double @llvm.vector.reduce.fdot.v2f64(double %acc, <2 x double> %a, <2 x double> %b)
+
+Overview:
+"""""""""
+
+The '``llvm.vector.reduce.fdot.*``' intrinsic computes the floating-point dot
+product of two vectors, returning the result as a scalar. The return type
+matches the element-type of the vector inputs.
+
+Arguments:
+""""""""""
+
+The first argument is the scalar start value (accumulator) of the reduction.
+The second and third arguments must be FP vectors of identical type.
+
+Semantics:
+""""""""""
+
+The intrinsic computes ``%acc + sum(%a[i] * %b[i])``. The order of floating-
+point operations and therefore the final result depends on the fast-math flags
+attached to the call:
+
+* Without any fast-math flags (default), the computation is performed as a
+ sequential chain of ``fmul``/``fadd`` pairs: each element of ``%a`` is
+ multiplied by the corresponding element of ``%b``, and the product is added
+ to the running accumulator. Two roundings occur per element (one for the
+ multiply, one for the add).
+
+* With the ``contract`` fast-math flag, the computation uses a sequential FMA
+ chain: ``fma(%a[0], %b[0], fma(%a[1], %b[1], ... %acc ...))``. Only one
+ rounding occurs per element.
+
+* With the ``reassoc`` fast-math flag, the order of operations is unspecified.
+ Targets may exploit this to perform a tree-based or otherwise reordered
+ reduction for improved performance.
+
+Examples:
+"""""""""
+
+::
+
+ %dot = call float @llvm.vector.reduce.fdot.v4f32(float 0.0, <4 x float> %a, <4 x float> %b)
+ %dot_fma = call contract float @llvm.vector.reduce.fdot.v4f32(float 0.0, <4 x float> %a, <4 x float> %b)
+ %dot_fast = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float 0.0, <4 x float> %a, <4 x float> %b)
+
.. _int_vector_reduce_and:
'``llvm.vector.reduce.and.*``' Intrinsic
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 8a8a9ee71ca02..c206f8e54cf0f 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1488,6 +1488,13 @@ enum NodeType {
VECREDUCE_SEQ_FADD,
VECREDUCE_SEQ_FMUL,
+ /// VECREDUCE_FDOT(Acc: scalar, VecA, VecB) - Floating-point dot product
+ /// reduction. Computes Acc + sum(VecA[i] * VecB[i]).
+ /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
+ /// With 'contract' flag: sequential FMA chain (single rounding per element).
+ /// With 'reassoc' flag: order of operations is unspecified.
+ VECREDUCE_FDOT,
+
/// These reductions have relaxed evaluation order semantics, and have a
/// single vector operand. The order of evaluation is unspecified. For
/// pow-of-2 vectors, one valid legalizer expansion is to use a tree
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 59a0f2d2e0c2a..b64c5c29886cc 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5863,6 +5863,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
+ /// Expand a VECREDUCE_FDOT into an explicit fmul+fadd or FMA chain.
+ /// Uses FMA chain when the 'contract' fast-math flag is set, otherwise
+ /// expands to a sequential fmul+fadd chain.
+ SDValue expandVecReduceDot(SDNode *Node, SelectionDAG &DAG) const;
+
/// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
/// Returns true if the expansion was successful.
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 80ef58f51e7bb..ffdeef7b19b4b 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2783,6 +2783,10 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrNoCreateUndefOrPoison] in
def int_vector_reduce_fmul : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
[LLVMVectorElementType<0>,
llvm_anyvector_ty]>;
+ def int_vector_reduce_fdot : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+ [LLVMVectorElementType<0>,
+ llvm_anyvector_ty,
+ LLVMMatchType<0>]>;
def int_vector_reduce_add : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
[llvm_anyvector_ty]>;
def int_vector_reduce_mul : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index f4a07e1988747..ec9d0c415a442 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -35,6 +35,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
default: break;
case Intrinsic::vector_reduce_fadd:
case Intrinsic::vector_reduce_fmul:
+ case Intrinsic::vector_reduce_fdot:
case Intrinsic::vector_reduce_add:
case Intrinsic::vector_reduce_mul:
case Intrinsic::vector_reduce_and:
@@ -87,6 +88,31 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
}
break;
}
+ case Intrinsic::vector_reduce_fdot: {
+ // Dot product: acc + sum(vecA[i] * vecB[i]).
+ // With 'contract': fma(a[i], b[i], rdx) chain.
+ // Without 'contract': sequential fmul(a[i], b[i]) + fadd(rdx, prod).
+ Value *Rdx = II->getArgOperand(0);
+ Value *VecA = II->getArgOperand(1);
+ Value *VecB = II->getArgOperand(2);
+ auto *VecTy = cast<FixedVectorType>(VecA->getType());
+ unsigned NumElts = VecTy->getNumElements();
+ for (unsigned i = 0; i < NumElts; i++) {
+ Value *Ai = Builder.CreateExtractElement(VecA, i);
+ Value *Bi = Builder.CreateExtractElement(VecB, i);
+ if (FMF.allowContract()) {
+ Rdx = Builder.CreateIntrinsic(
+ Intrinsic::fma, {VecTy->getElementType()}, {Ai, Bi, Rdx});
+ } else {
+ Value *Prod = Builder.CreateFMul(Ai, Bi);
+ Rdx = Builder.CreateFAdd(Rdx, Prod);
+ }
+ }
+ II->replaceAllUsesWith(Rdx);
+ II->eraseFromParent();
+ Changed = true;
+ continue;
+ }
case Intrinsic::vector_reduce_and:
case Intrinsic::vector_reduce_or: {
// Canonicalize logical or/and reductions:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 89daec05f7135..76a8ae38daaa8 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2571,6 +2571,27 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
return true;
}
+ case Intrinsic::vector_reduce_fdot: {
+ // Lower as G_FMUL(vecA, vecB) followed by G_VECREDUCE_SEQ_FADD or
+ // G_VECREDUCE_FADD + G_FADD depending on the reassoc flag.
+ Register Dst = getOrCreateVReg(CI);
+ Register AccSrc = getOrCreateVReg(*CI.getArgOperand(0));
+ Register VecA = getOrCreateVReg(*CI.getArgOperand(1));
+ Register VecB = getOrCreateVReg(*CI.getArgOperand(2));
+ LLT VecTy = MRI->getType(VecA);
+ auto MIFlags = MachineInstr::copyFlagsFromInstruction(CI);
+ auto Prod = MIRBuilder.buildFMul(VecTy, VecA, VecB, MIFlags);
+ if (!CI.hasAllowReassoc()) {
+ MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FADD, {Dst},
+ {AccSrc, Prod}, MIFlags);
+ } else {
+ LLT DstTy = MRI->getType(Dst);
+ auto Rdx = MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_FADD, {DstTy},
+ {Prod}, MIFlags);
+ MIRBuilder.buildFAdd(Dst, AccSrc, Rdx, MIFlags);
+ }
+ return true;
+ }
case Intrinsic::trap:
return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP);
case Intrinsic::debugtrap:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 25f4f75eaedea..68de7a165cac9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -179,6 +179,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
+ case ISD::VECREDUCE_FDOT: R = SoftenFloatRes_VECREDUCE_FDOT(N); break;
// clang-format on
}
@@ -1145,6 +1146,11 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
return SDValue();
}
+SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_FDOT(SDNode *N) {
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
+ return SDValue();
+}
+
//===----------------------------------------------------------------------===//
// Convert Float Operand to Integer
//===----------------------------------------------------------------------===//
@@ -2786,6 +2792,9 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_SEQ_FMUL:
R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_FDOT:
+ R = SoftPromoteHalfRes_VECREDUCE_FDOT(N);
+ break;
}
if (R.getNode())
@@ -3142,6 +3151,12 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
return SDValue();
}
+SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_FDOT(SDNode *N) {
+ // Expand and soften.
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
+ return SDValue();
+}
+
//===----------------------------------------------------------------------===//
// Half Operand Soft Promotion
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 84c91a80ade79..d06d7494b4033 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -630,6 +630,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
SDValue SoftenFloatRes_VECREDUCE(SDNode *N);
SDValue SoftenFloatRes_VECREDUCE_SEQ(SDNode *N);
+ SDValue SoftenFloatRes_VECREDUCE_FDOT(SDNode *N);
// Convert Float Operand to Integer.
bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
@@ -796,6 +797,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N);
+ SDValue SoftPromoteHalfRes_VECREDUCE_FDOT(SDNode *N);
bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_BITCAST(SDNode *N);
@@ -882,6 +884,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N);
SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
SDValue ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue ScalarizeVecOp_CMP(SDNode *N);
SDValue ScalarizeVecOp_FAKE_USE(SDNode *N);
SDValue ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
@@ -971,6 +974,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue SplitVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_UnaryOp(SDNode *N);
SDValue SplitVecOp_TruncateHelper(SDNode *N);
@@ -1117,6 +1121,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecOp_IS_FPCLASS(SDNode *N);
SDValue WidenVecOp_VECREDUCE(SDNode *N);
SDValue WidenVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue WidenVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue WidenVecOp_VP_REDUCE(SDNode *N);
SDValue WidenVecOp_ExpOp(SDNode *N);
SDValue WidenVecOp_VP_CttzElements(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index ccad9354fc820..7c42ef2a9861f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -531,6 +531,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
+ case ISD::VECREDUCE_FDOT:
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
@@ -1316,6 +1317,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
+ case ISD::VECREDUCE_FDOT:
+ Results.push_back(TLI.expandVecReduceDot(Node, DAG));
+ return;
case ISD::SREM:
case ISD::UREM:
ExpandREM(Node, Results);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index a96c77bc6a4e9..046caa50e0248 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -933,6 +933,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_FDOT:
+ Res = ScalarizeVecOp_VECREDUCE_FDOT(N);
+ break;
case ISD::SCMP:
case ISD::UCMP:
Res = ScalarizeVecOp_CMP(N);
@@ -1241,6 +1244,24 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
AccOp, Op, N->getFlags());
}
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N) {
+ SDLoc dl(N);
+ SDValue AccOp = N->getOperand(0);
+ SDValue VecAOp = N->getOperand(1);
+ SDValue VecBOp = N->getOperand(2);
+ SDNodeFlags Flags = N->getFlags();
+ EVT EltVT = N->getValueType(0);
+
+ SDValue A = GetScalarizedVector(VecAOp);
+ SDValue B = GetScalarizedVector(VecBOp);
+
+ if (Flags.hasAllowContract())
+ return DAG.getNode(ISD::FMA, dl, EltVT, A, B, AccOp, Flags);
+
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, A, B, Flags);
+ return DAG.getNode(ISD::FADD, dl, EltVT, AccOp, Mul, Flags);
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(0));
SDValue RHS = GetScalarizedVector(N->getOperand(1));
@@ -3817,6 +3838,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = SplitVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_FDOT:
+ Res = SplitVecOp_VECREDUCE_FDOT(N);
+ break;
case ISD::VP_REDUCE_FADD:
case ISD::VP_REDUCE_SEQ_FADD:
case ISD::VP_REDUCE_FMUL:
@@ -3991,6 +4015,27 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
}
+SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_FDOT(SDNode *N) {
+ EVT ResVT = N->getValueType(0);
+ SDLoc dl(N);
+
+ SDValue AccOp = N->getOperand(0);
+ SDValue VecAOp = N->getOperand(1);
+ SDValue VecBOp = N->getOperand(2);
+ SDNodeFlags Flags = N->getFlags();
+
+ assert(VecAOp.getValueType().isVector() && "Expected vector operand");
+ SDValue LoA, HiA, LoB, HiB;
+ GetSplitVector(VecAOp, LoA, HiA);
+ GetSplitVector(VecBOp, LoB, HiB);
+
+ // Reduce the lower half first, using the original accumulator.
+ SDValue Partial =
+ DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, AccOp, LoA, LoB, Flags);
+ // Reduce the upper half, using the lower-half result as the new accumulator.
+ return DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, Partial, HiA, HiB, Flags);
+}
+
SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
assert(N->isVPOpcode() && "Expected VP opcode");
assert(OpNo == 1 && "Can only split reduce vector operand");
@@ -7473,6 +7518,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = WidenVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_FDOT:
+ Res = WidenVecOp_VECREDUCE_FDOT(N);
+ break;
case ISD::VP_REDUCE_FADD:
case ISD::VP_REDUCE_SEQ_FADD:
case ISD::VP_REDUCE_FMUL:
@@ -8410,6 +8458,36 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
return DAG.getNode(Opc, dl, VT, AccOp, Op, Flags);
}
+SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_FDOT(SDNode *N) {
+ SDLoc dl(N);
+ SDValue AccOp = N->getOperand(0);
+ SDValue VecAOp = N->getOperand(1);
+ SDValue VecBOp = N->getOperand(2);
+
+ SDValue OpA = GetWidenedVector(VecAOp);
+ SDValue OpB = GetWidenedVector(VecBOp);
+
+ EVT VT = N->getValueType(0);
+ EVT OrigVT = VecAOp.getValueType();
+ EVT WideVT = OpA.getValueType();
+ EVT ElemVT = OrigVT.getVectorElementType();
+ SDNodeFlags Flags = N->getFlags();
+
+ // Pad the extra lanes of both vectors with 0.0.
+ // FMA(0, 0, acc) = acc, so padding with zeros is correct.
+ SDValue Zero = DAG.getConstantFP(0.0, dl, ElemVT);
+
+ unsigned OrigElts = OrigVT.getVectorMinNumElements();
+ unsigned WideElts = WideVT.getVectorMinNumElements();
+
+ for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) {
+ OpA = DAG.getInsertVectorElt(dl, OpA, Zero, Idx);
+ OpB = DAG.getInsertVectorElt(dl, OpB, Zero, Idx);
+ }
+
+ return DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, AccOp, OpA, OpB, Flags);
+}
+
SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
assert(N->isVPOpcode() && "Expected VP opcode");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1f3b099c9c577..eb3ec8aa8495e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8045,6 +8045,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::vector_reduce_fmin:
case Intrinsic::vector_reduce_fmaximum:
case Intrinsic::vector_reduce_fminimum:
+ case Intrinsic::vector_reduce_fdot:
visitVectorReduce(I, Intrinsic);
return;
@@ -11142,6 +11143,9 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
SDValue Op2;
if (I.arg_size() > 1)
Op2 = getValue(I.getArgOperand(1));
+ SDValue Op3;
+ if (I.arg_size() > 2)
+ Op3 = getValue(I.getArgOperand(2));
SDLoc dl = getCurSDLoc();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
SDValue Res;
@@ -11205,6 +11209,11 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
case Intrinsic::vector_reduce_fminimum:
Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
break;
+ case Intrinsic::vector_reduce_fdot:
+ // Always emit VECREDUCE_FDOT; all FMF flags (contract, reassoc, etc.)
+ // are carried in SDFlags and checked during expansion.
+ Res = DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, {Op1, Op2, Op3}, SDFlags);
+ break;
default:
llvm_unreachable("Unhandled vector reduce intrinsic");
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index ce78072d21114..7aad3023bf484 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -568,6 +568,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::VECREDUCE_SEQ_FADD: return "vecreduce_seq_fadd";
case ISD::VECREDUCE_FMUL: return "vecreduce_fmul";
case ISD::VECREDUCE_SEQ_FMUL: return "vecreduce_seq_fmul";
+ case ISD::VECREDUCE_FDOT: return "vecreduce_fdot";
case ISD::VECREDUCE_ADD: return "vecreduce_add";
case ISD::VECREDUCE_MUL: return "vecreduce_mul";
case ISD::VECREDUCE_AND: return "vecreduce_and";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index e6aa222425d13..ff6d259528cac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12259,6 +12259,43 @@ SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) cons
return Res;
}
+SDValue TargetLowering::expandVecReduceDot(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ SDValue AccOp = Node->getOperand(0);
+ SDValue VecAOp = Node->getOperand(1);
+ SDValue VecBOp = Node->getOperand(2);
+ SDNodeFlags Flags = Node->getFlags();
+
+ EVT VT = VecAOp.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+
+ if (VT.isScalableVector())
+ report_fatal_error(
+ "Expanding reductions for scalable vectors is undefined.");
+
+ unsigned NumElts = VT.getVectorNumElements();
+
+ SmallVector<SDValue, 8> OpsA, OpsB;
+ DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
+ DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
+
+ SDValue Res = AccOp;
+ if (Flags.hasAllowContract()) {
+ // With 'contract': use FMA chain for single-rounded dot product.
+ for (unsigned i = 0; i < NumElts; i++)
+ Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
+ } else {
+ // Default: sequential fmul + fadd (two roundings per element).
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
+ }
+
+ return Res;
+}
+
bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 2f1e3f2f3ff7a..daa4b7e22dc67 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1221,7 +1221,8 @@ void TargetLoweringBase::initActions() {
ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN, ISD::VECREDUCE_FMAX,
ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM,
- ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_SEQ_FMUL},
+ ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_SEQ_FMUL,
+ ISD::VECREDUCE_FDOT},
VT, Expand);
// Named vector shuffles default to expand.
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index d4ade9c7ce534..69206b1fdc3b5 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6620,6 +6620,16 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"Intrinsic has incorrect argument type!");
break;
}
+ case Intrinsic::vector_reduce_fdot: {
+ // First arg is scalar accumulator; second and third are FP vectors of the
+ // same type.
+ Type *ArgTy = Call.getArgOperand(1)->getType();
+ Check(ArgTy->isFPOrFPVectorTy() && ArgTy->isVectorTy(),
+ "Intrinsic has incorrect argument type!");
+ Check(Call.getArgOperand(2)->getType() == ArgTy,
+ "Intrinsic vector arguments must have the same type!");
+ break;
+ }
case Intrinsic::smul_fix:
case Intrinsic::smul_fix_sat:
case Intrinsic::umul_fix:
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
new file mode 100644
index 0000000000000..5cf90cdc2c56c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -O0 | FileCheck %s -check-prefix=O0
+; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -O1 | FileCheck %s -check-prefix=O1
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
+
+define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; O0-LABEL: fdot_f32:
+; O0: // %bb.0:
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: fmov s4, s1
+; O0-NEXT: mov s3, v1.s[1]
+; O0-NEXT: mov s2, v1.s[2]
+; O0-NEXT: mov s1, v1.s[3]
+; O0-NEXT: fadd s0, s0, s4
+; O0-NEXT: fadd s0, s0, s3
+; O0-NEXT: fadd s0, s0, s2
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fmov w0, s0
+; O0-NEXT: fmov s0, w0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f32:
+; O1: // %bb.0:
+; O1-NEXT: mov s3, v1.s[1]
+; O1-NEXT: fmul s4, s1, v2.s[0]
+; O1-NEXT: mov s5, v1.s[2]
+; O1-NEXT: mov s1, v1.s[3]
+; O1-NEXT: fmul s3, s3, v2.s[1]
+; O1-NEXT: fadd s0, s0, s4
+; O1-NEXT: fmul s1, s1, v2.s[3]
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fmul s3, s5, v2.s[2]
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: ret
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; O0-LABEL: fdot_f32_contract:
+; O0: // %bb.0:
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: fmov s4, s1
+; O0-NEXT: mov s3, v1.s[1]
+; O0-NEXT: mov s2, v1.s[2]
+; O0-NEXT: mov s1, v1.s[3]
+; O0-NEXT: fadd s0, s0, s4
+; O0-NEXT: fadd s0, s0, s3
+; O0-NEXT: fadd s0, s0, s2
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fmov w0, s0
+; O0-NEXT: fmov s0, w0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f32_contract:
+; O1: // %bb.0:
+; O1-NEXT: mov s3, v2.s[1]
+; O1-NEXT: fmadd s0, s1, s2, s0
+; O1-NEXT: mov s4, v2.s[2]
+; O1-NEXT: mov s2, v2.s[3]
+; O1-NEXT: fmla s0, s3, v1.s[1]
+; O1-NEXT: fmla s0, s4, v1.s[2]
+; O1-NEXT: fmla s0, s2, v1.s[3]
+; O1-NEXT: ret
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
+; O0-LABEL: fdot_f16:
+; O0: // %bb.0:
+; O0-NEXT: fcvtl v1.4s, v1.4h
+; O0-NEXT: fcvtl v2.4s, v2.4h
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: fcvtn v5.4h, v1.4s
+; O0-NEXT: fmov s4, s5
+; O0-NEXT: // implicit-def: $q1
+; O0-NEXT: fmov d1, d5
+; O0-NEXT: mov h3, v1.h[1]
+; O0-NEXT: // implicit-def: $q1
+; O0-NEXT: fmov d1, d5
+; O0-NEXT: mov h2, v1.h[2]
+; O0-NEXT: // implicit-def: $q1
+; O0-NEXT: fmov d1, d5
+; O0-NEXT: mov h1, v1.h[3]
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s4, h4
+; O0-NEXT: fadd s0, s0, s4
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fadd s0, s0, s3
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s2, h2
+; O0-NEXT: fadd s0, s0, s2
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: // kill: def $s0 killed $h0
+; O0-NEXT: fmov w0, s0
+; O0-NEXT: fmov s0, w0
+; O0-NEXT: // kill: def $h0 killed $h0 killed $s0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f16:
+; O1: // %bb.0:
+; O1-NEXT: // kill: def $d2 killed $d2 def $q2
+; O1-NEXT: // kill: def $d1 killed $d1 def $q1
+; O1-NEXT: fcvt s3, h2
+; O1-NEXT: fcvt s4, h1
+; O1-NEXT: mov h5, v2.h[1]
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul s3, s4, s3
+; O1-NEXT: mov h4, v1.h[1]
+; O1-NEXT: fcvt s5, h5
+; O1-NEXT: fcvt h3, s3
+; O1-NEXT: fcvt s4, h4
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fmul s4, s4, s5
+; O1-NEXT: mov h5, v2.h[2]
+; O1-NEXT: mov h2, v2.h[3]
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fcvt h3, s4
+; O1-NEXT: mov h4, v1.h[2]
+; O1-NEXT: fcvt s5, h5
+; O1-NEXT: mov h1, v1.h[3]
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fcvt s4, h4
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul s1, s1, s2
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fmul s3, s4, s5
+; O1-NEXT: fcvt h1, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt h3, s3
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: ret
+ %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
new file mode 100644
index 0000000000000..64964ab8c37ac
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+
+; Default: sequential v_mul_f32 + v_add_f32 chain.
+define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; GFX9-LABEL: fdot_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v2, v6
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v7
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v8
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract: sequential v_fma_f32 chain.
+define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; GFX9-LABEL: fdot_f32_contract:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v1, v5, v0
+; GFX9-NEXT: v_fma_f32 v0, v2, v6, v0
+; GFX9-NEXT: v_fma_f32 v0, v3, v7, v0
+; GFX9-NEXT: v_fma_f32 v0, v4, v8, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
new file mode 100644
index 0000000000000..e761911126b59
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=expand-reductions -S | FileCheck %s
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare float @llvm.fma.f32(float, float, float)
+
+; Default (no fast-math flags): sequential fmul + fadd chain.
+define float @fdot_ordered(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @fdot_ordered(
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[PROD0:%.*]] = fmul float [[A0]], [[B0]]
+; CHECK-NEXT: [[ACC0:%.*]] = fadd float [[ACC:%.*]], [[PROD0]]
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i64 1
+; CHECK-NEXT: [[PROD1:%.*]] = fmul float [[A1]], [[B1]]
+; CHECK-NEXT: [[ACC1:%.*]] = fadd float [[ACC0]], [[PROD1]]
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i64 2
+; CHECK-NEXT: [[PROD2:%.*]] = fmul float [[A2]], [[B2]]
+; CHECK-NEXT: [[ACC2:%.*]] = fadd float [[ACC1]], [[PROD2]]
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i64 3
+; CHECK-NEXT: [[PROD3:%.*]] = fmul float [[A3]], [[B3]]
+; CHECK-NEXT: [[RDX:%.*]] = fadd float [[ACC2]], [[PROD3]]
+; CHECK-NEXT: ret float [[RDX]]
+;
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract flag: sequential FMA chain.
+define float @fdot_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @fdot_contract(
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[RDX0:%.*]] = call contract float @llvm.fma.f32(float [[A0]], float [[B0]], float [[ACC:%.*]])
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i64 1
+; CHECK-NEXT: [[RDX1:%.*]] = call contract float @llvm.fma.f32(float [[A1]], float [[B1]], float [[RDX0]])
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i64 2
+; CHECK-NEXT: [[RDX2:%.*]] = call contract float @llvm.fma.f32(float [[A2]], float [[B2]], float [[RDX1]])
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i64 3
+; CHECK-NEXT: [[RDX:%.*]] = call contract float @llvm.fma.f32(float [[A3]], float [[B3]], float [[RDX2]])
+; CHECK-NEXT: ret float [[RDX]]
+;
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll b/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
new file mode 100644
index 0000000000000..90deab32f7759
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
+; RUN: -disable-post-ra -verify-machineinstrs \
+; RUN: | FileCheck %s
+target triple = "nvptx64-nvidia-cuda"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
+
+; Default: sequential mul + add chain.
+define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<18>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_param_2];
+; CHECK-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_param_1];
+; CHECK-NEXT: ld.param.b32 %r1, [fdot_f32_param_0];
+; CHECK-NEXT: mul.rn.f32 %r10, %r3, %r7;
+; CHECK-NEXT: mul.rn.f32 %r11, %r2, %r6;
+; CHECK-NEXT: add.rn.f32 %r12, %r1, %r11;
+; CHECK-NEXT: add.rn.f32 %r13, %r12, %r10;
+; CHECK-NEXT: mul.rn.f32 %r14, %r4, %r8;
+; CHECK-NEXT: add.rn.f32 %r15, %r13, %r14;
+; CHECK-NEXT: mul.rn.f32 %r16, %r5, %r9;
+; CHECK-NEXT: add.rn.f32 %r17, %r15, %r16;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-NEXT: ret;
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract: sequential fma chain.
+define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32_contract(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<14>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_contract_param_2];
+; CHECK-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_contract_param_1];
+; CHECK-NEXT: ld.param.b32 %r1, [fdot_f32_contract_param_0];
+; CHECK-NEXT: fma.rn.f32 %r10, %r2, %r6, %r1;
+; CHECK-NEXT: fma.rn.f32 %r11, %r3, %r7, %r10;
+; CHECK-NEXT: fma.rn.f32 %r12, %r4, %r8, %r11;
+; CHECK-NEXT: fma.rn.f32 %r13, %r5, %r9, %r12;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT: ret;
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; f16 variant: sequential mul + add chain.
+define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<18>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_param_1];
+; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_param_0];
+; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r1;
+; CHECK-NEXT: mov.b32 {%rs4, %rs5}, %r3;
+; CHECK-NEXT: mul.rn.f16 %rs6, %rs3, %rs5;
+; CHECK-NEXT: mul.rn.f16 %rs7, %rs2, %rs4;
+; CHECK-NEXT: add.rn.f16 %rs8, %rs1, %rs7;
+; CHECK-NEXT: add.rn.f16 %rs9, %rs8, %rs6;
+; CHECK-NEXT: mov.b32 {%rs10, %rs11}, %r2;
+; CHECK-NEXT: mov.b32 {%rs12, %rs13}, %r4;
+; CHECK-NEXT: mul.rn.f16 %rs14, %rs10, %rs12;
+; CHECK-NEXT: add.rn.f16 %rs15, %rs9, %rs14;
+; CHECK-NEXT: mul.rn.f16 %rs16, %rs11, %rs13;
+; CHECK-NEXT: add.rn.f16 %rs17, %rs15, %rs16;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs17;
+; CHECK-NEXT: ret;
+ %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
new file mode 100644
index 0000000000000..90eeb1a5eee31
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp,+fp64 -verify-machineinstrs %s -o - | FileCheck %s
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
+
+; Default: sequential vmla.f32 chain (mul + accumulate).
+define arm_aapcs_vfpcc float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vmla.f32 s0, s4, s8
+; CHECK-NEXT: vmla.f32 s0, s5, s9
+; CHECK-NEXT: vmla.f32 s0, s6, s10
+; CHECK-NEXT: vmla.f32 s0, s7, s11
+; CHECK-NEXT: bx lr
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract: vfma.f32 chain.
+define arm_aapcs_vfpcc float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32_contract:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vfma.f32 s0, s4, s8
+; CHECK-NEXT: vfma.f32 s0, s5, s9
+; CHECK-NEXT: vfma.f32 s0, s6, s10
+; CHECK-NEXT: vfma.f32 s0, s7, s11
+; CHECK-NEXT: bx lr
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
new file mode 100644
index 0000000000000..f40945bdb6382
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fma | FileCheck %s --check-prefix=FMA
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+
+; Default: sequential vmulss + vaddss chain.
+define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; AVX2-LABEL: fdot_f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX2-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; FMA-LABEL: fdot_f32:
+; FMA: # %bb.0:
+; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; FMA-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; FMA-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
+; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; FMA-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; FMA-NEXT: retq
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract and FMA hardware: vfmadd chain.
+define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; AVX2-LABEL: fdot_f32_contract:
+; AVX2: # %bb.0:
+; AVX2-NEXT: subq $40, %rsp
+; AVX2-NEXT: .cfi_def_cfa_offset 48
+; AVX2-NEXT: vmovaps %xmm2, %xmm3
+; AVX2-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill
+; AVX2-NEXT: vmovaps %xmm0, %xmm2
+; AVX2-NEXT: vmovaps %xmm1, %xmm0
+; AVX2-NEXT: vmovaps %xmm3, %xmm1
+; AVX2-NEXT: callq fmaf at PLT
+; AVX2-NEXT: vmovaps %xmm0, %xmm2
+; AVX2-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[1,1,3,3]
+; AVX2-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm1 = mem[1,1,3,3]
+; AVX2-NEXT: callq fmaf at PLT
+; AVX2-NEXT: vmovaps %xmm0, %xmm2
+; AVX2-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[1,0]
+; AVX2-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm1 = mem[1,0]
+; AVX2-NEXT: callq fmaf at PLT
+; AVX2-NEXT: vmovapd %xmm0, %xmm2
+; AVX2-NEXT: vpermilps $255, (%rsp), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm0 = mem[3,3,3,3]
+; AVX2-NEXT: vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT: # xmm1 = mem[3,3,3,3]
+; AVX2-NEXT: addq $40, %rsp
+; AVX2-NEXT: .cfi_def_cfa_offset 8
+; AVX2-NEXT: jmp fmaf at PLT # TAILCALL
+;
+; FMA-LABEL: fdot_f32_contract:
+; FMA: # %bb.0:
+; FMA-NEXT: vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0
+; FMA-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; FMA-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; FMA-NEXT: vfmadd213ss {{.*#+}} xmm4 = (xmm3 * xmm4) + xmm0
+; FMA-NEXT: vshufpd {{.*#+}} xmm0 = xmm1[1,0]
+; FMA-NEXT: vshufpd {{.*#+}} xmm3 = xmm2[1,0]
+; FMA-NEXT: vfmadd213ss {{.*#+}} xmm3 = (xmm0 * xmm3) + xmm4
+; FMA-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; FMA-NEXT: vshufps {{.*#+}} xmm0 = xmm2[3,3,3,3]
+; FMA-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm3
+; FMA-NEXT: retq
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
>From 2f2f0de370e6c6ac02bbdf4ddaeac0297ffb60ad Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 02:51:28 -0700
Subject: [PATCH 02/17] Split VECREDUCE_FDOT into VECREDUCE_SEQ_FDOT +
VECREDUCE_FDOT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Mirror the VECREDUCE_SEQ_FADD / VECREDUCE_FADD split for dot products:
- VECREDUCE_SEQ_FDOT(Acc, VecA, VecB) — 3-operand, ordered/sequential
(emitted when the call has no 'reassoc' flag)
- VECREDUCE_FDOT(VecA, VecB) — 2-operand, unordered, no accumulator
(emitted when 'reassoc' is set; acc added via wrapping FADD)
SelectionDAGBuilder dispatches on the 'reassoc' flag exactly like
VECREDUCE_SEQ_FADD / VECREDUCE_FADD. Both nodes default to Expand;
LegalizeVectorOps inlines the expansions:
- SEQ: FMA (contract) or FMUL+FADD chain left-to-right with acc.
Cannot reuse expandVecReduceSeq — fdot has no single base opcode.
Remove expandVecReduceDot from TargetLowering for the same reason.
- Non-SEQ: FMUL(vecA, vecB) → VECREDUCE_FADD(products), reusing the
existing unordered-reduce infrastructure.
LegalizeVectorTypes and LegalizeFloatTypes updated with separate
Scalarize/Split/Widen/Soften/SoftPromoteHalf handlers for both nodes.
GlobalISel IRTranslator already implemented the split correctly.
ExpandReductions updated for the IR-level reassoc path: vector fmul
then shuffle-tree fadd (power-of-2), else sequential fallback.
Test coverage:
- All existing fdot test files gain a 'reassoc' variant.
- NVPTX vecreduce-fdot.ll merged into reduction-intrinsics.ll (adds
sm_100 coverage; sm_100 uses mul.rn.f32x2 for the reassoc path).
- Generic expand-fdot-reduction.ll gains a 'reassoc' shuffle-tree case.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 14 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 5 -
llvm/lib/CodeGen/ExpandReductions.cpp | 37 ++--
.../SelectionDAG/LegalizeFloatTypes.cpp | 79 ++++++++-
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 5 +
.../SelectionDAG/LegalizeVectorOps.cpp | 41 ++++-
.../SelectionDAG/LegalizeVectorTypes.cpp | 90 ++++++++--
.../SelectionDAG/SelectionDAGBuilder.cpp | 11 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 1 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 37 ----
llvm/lib/CodeGen/TargetLoweringBase.cpp | 2 +-
llvm/test/CodeGen/AArch64/vecreduce-fdot.ll | 21 +++
.../test/CodeGen/AMDGPU/vector-reduce-fdot.ll | 18 ++
.../CodeGen/Generic/expand-fdot-reduction.ll | 16 ++
.../CodeGen/NVPTX/reduction-intrinsics.ll | 164 ++++++++++++++++++
llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll | 82 ---------
.../test/CodeGen/Thumb2/mve-vecreduce-fdot.ll | 14 ++
llvm/test/CodeGen/X86/vector-reduce-fdot.ll | 41 +++++
18 files changed, 514 insertions(+), 164 deletions(-)
delete mode 100644 llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index c206f8e54cf0f..33e14e739014a 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1487,13 +1487,12 @@ enum NodeType {
/// RES = (((ACC + SRC_VEC[0]) + SRC_VEC[1]) + SRC_VEC[2]) + SRC_VEC[3]
VECREDUCE_SEQ_FADD,
VECREDUCE_SEQ_FMUL,
-
- /// VECREDUCE_FDOT(Acc: scalar, VecA, VecB) - Floating-point dot product
- /// reduction. Computes Acc + sum(VecA[i] * VecB[i]).
+ /// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
+ /// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
+ /// left-to-right order.
/// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
/// With 'contract' flag: sequential FMA chain (single rounding per element).
- /// With 'reassoc' flag: order of operations is unspecified.
- VECREDUCE_FDOT,
+ VECREDUCE_SEQ_FDOT,
/// These reductions have relaxed evaluation order semantics, and have a
/// single vector operand. The order of evaluation is unspecified. For
@@ -1516,6 +1515,11 @@ enum NodeType {
/// llvm.minimum and llvm.maximum semantics.
VECREDUCE_FMAXIMUM,
VECREDUCE_FMINIMUM,
+ /// VECREDUCE_FDOT(VecA, VecB) - Unordered floating-point dot product
+ /// reduction. Computes sum(VecA[i] * VecB[i]) with unspecified evaluation
+ /// order. The caller adds the accumulator via a separate FADD. Decomposes
+ /// to FMUL(VecA, VecB) followed by VECREDUCE_FADD on the products.
+ VECREDUCE_FDOT,
/// Integer reductions may have a result type larger than the vector element
/// type. However, the reduction is performed using the vector element type
/// and the value in the top bits is unspecified.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index b64c5c29886cc..59a0f2d2e0c2a 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5863,11 +5863,6 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
- /// Expand a VECREDUCE_FDOT into an explicit fmul+fadd or FMA chain.
- /// Uses FMA chain when the 'contract' fast-math flag is set, otherwise
- /// expands to a sequential fmul+fadd chain.
- SDValue expandVecReduceDot(SDNode *Node, SelectionDAG &DAG) const;
-
/// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
/// Returns true if the expansion was successful.
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index ec9d0c415a442..748fc1229f2ea 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -90,25 +90,36 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
}
case Intrinsic::vector_reduce_fdot: {
// Dot product: acc + sum(vecA[i] * vecB[i]).
- // With 'contract': fma(a[i], b[i], rdx) chain.
- // Without 'contract': sequential fmul(a[i], b[i]) + fadd(rdx, prod).
- Value *Rdx = II->getArgOperand(0);
+ Value *Acc = II->getArgOperand(0);
Value *VecA = II->getArgOperand(1);
Value *VecB = II->getArgOperand(2);
auto *VecTy = cast<FixedVectorType>(VecA->getType());
unsigned NumElts = VecTy->getNumElements();
- for (unsigned i = 0; i < NumElts; i++) {
- Value *Ai = Builder.CreateExtractElement(VecA, i);
- Value *Bi = Builder.CreateExtractElement(VecB, i);
- if (FMF.allowContract()) {
- Rdx = Builder.CreateIntrinsic(
- Intrinsic::fma, {VecTy->getElementType()}, {Ai, Bi, Rdx});
- } else {
- Value *Prod = Builder.CreateFMul(Ai, Bi);
- Rdx = Builder.CreateFAdd(Rdx, Prod);
+ Value *LocalRdx;
+ if (FMF.allowReassoc() && isPowerOf2_32(NumElts)) {
+ // Reassoc + power-of-2: fmul vector, shuffle-tree fadd, then add acc.
+ Value *Products = Builder.CreateFMul(VecA, VecB);
+ LocalRdx = getShuffleReduction(Builder, Products, Instruction::FAdd,
+ RS, RK);
+ LocalRdx = Builder.CreateBinOp(Instruction::FAdd, Acc, LocalRdx,
+ "bin.rdx");
+ } else {
+ // Sequential: with 'contract': fma(a[i], b[i], rdx) chain.
+ // without: fmul(a[i], b[i]) + fadd(rdx, prod) chain.
+ LocalRdx = Acc;
+ for (unsigned i = 0; i < NumElts; i++) {
+ Value *Ai = Builder.CreateExtractElement(VecA, i);
+ Value *Bi = Builder.CreateExtractElement(VecB, i);
+ if (FMF.allowContract()) {
+ LocalRdx = Builder.CreateIntrinsic(
+ Intrinsic::fma, {VecTy->getElementType()}, {Ai, Bi, LocalRdx});
+ } else {
+ Value *Prod = Builder.CreateFMul(Ai, Bi);
+ LocalRdx = Builder.CreateFAdd(LocalRdx, Prod);
+ }
}
}
- II->replaceAllUsesWith(Rdx);
+ II->replaceAllUsesWith(LocalRdx);
II->eraseFromParent();
Changed = true;
continue;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 68de7a165cac9..ef5faa17fee27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -179,6 +179,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
+ case ISD::VECREDUCE_SEQ_FDOT: R = SoftenFloatRes_VECREDUCE_SEQ_FDOT(N); break;
case ISD::VECREDUCE_FDOT: R = SoftenFloatRes_VECREDUCE_FDOT(N); break;
// clang-format on
}
@@ -1146,8 +1147,43 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
return SDValue();
}
+SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ_FDOT(SDNode *N) {
+ // Inline the sequential expansion (VECREDUCE_SEQ_FDOT has acc in operand 0).
+ SDLoc dl(N);
+ SDValue AccOp = N->getOperand(0);
+ SDValue VecAOp = N->getOperand(1);
+ SDValue VecBOp = N->getOperand(2);
+ SDNodeFlags Flags = N->getFlags();
+ EVT VT = VecAOp.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ unsigned NumElts = VT.getVectorNumElements();
+ SmallVector<SDValue, 8> OpsA, OpsB;
+ DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
+ DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
+ SDValue Res = AccOp;
+ if (Flags.hasAllowContract())
+ for (unsigned i = 0; i < NumElts; i++)
+ Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
+ else
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
+ ReplaceValueWith(SDValue(N, 0), Res);
+ return SDValue();
+}
+
SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_FDOT(SDNode *N) {
- ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
+ // Decompose unordered FDOT to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
+ SDLoc dl(N);
+ SDValue VecAOp = N->getOperand(0);
+ SDValue VecBOp = N->getOperand(1);
+ SDNodeFlags Flags = N->getFlags();
+ EVT VT = VecAOp.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecAOp, VecBOp, Flags);
+ ReplaceValueWith(SDValue(N, 0),
+ DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
return SDValue();
}
@@ -2792,6 +2828,9 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::VECREDUCE_SEQ_FMUL:
R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_SEQ_FDOT:
+ R = SoftPromoteHalfRes_VECREDUCE_SEQ_FDOT(N);
+ break;
case ISD::VECREDUCE_FDOT:
R = SoftPromoteHalfRes_VECREDUCE_FDOT(N);
break;
@@ -3151,9 +3190,43 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
return SDValue();
}
+SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ_FDOT(SDNode *N) {
+ // Inline sequential expansion for the SEQ variant (acc in operand 0).
+ SDLoc dl(N);
+ SDValue AccOp = N->getOperand(0);
+ SDValue VecAOp = N->getOperand(1);
+ SDValue VecBOp = N->getOperand(2);
+ SDNodeFlags Flags = N->getFlags();
+ EVT VT = VecAOp.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ unsigned NumElts = VT.getVectorNumElements();
+ SmallVector<SDValue, 8> OpsA, OpsB;
+ DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
+ DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
+ SDValue Res = AccOp;
+ if (Flags.hasAllowContract())
+ for (unsigned i = 0; i < NumElts; i++)
+ Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
+ else
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
+ ReplaceValueWith(SDValue(N, 0), Res);
+ return SDValue();
+}
+
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_FDOT(SDNode *N) {
- // Expand and soften.
- ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
+ // Decompose unordered FDOT to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
+ SDLoc dl(N);
+ SDValue VecAOp = N->getOperand(0);
+ SDValue VecBOp = N->getOperand(1);
+ SDNodeFlags Flags = N->getFlags();
+ EVT VT = VecAOp.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecAOp, VecBOp, Flags);
+ ReplaceValueWith(SDValue(N, 0),
+ DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
return SDValue();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index d06d7494b4033..19d4d0005e6ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -630,6 +630,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
SDValue SoftenFloatRes_VECREDUCE(SDNode *N);
SDValue SoftenFloatRes_VECREDUCE_SEQ(SDNode *N);
+ SDValue SoftenFloatRes_VECREDUCE_SEQ_FDOT(SDNode *N);
SDValue SoftenFloatRes_VECREDUCE_FDOT(SDNode *N);
// Convert Float Operand to Integer.
@@ -797,6 +798,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N);
+ SDValue SoftPromoteHalfRes_VECREDUCE_SEQ_FDOT(SDNode *N);
SDValue SoftPromoteHalfRes_VECREDUCE_FDOT(SDNode *N);
bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);
@@ -884,6 +886,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N);
SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
SDValue ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue ScalarizeVecOp_VECREDUCE_SEQ_FDOT(SDNode *N);
SDValue ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue ScalarizeVecOp_CMP(SDNode *N);
SDValue ScalarizeVecOp_FAKE_USE(SDNode *N);
@@ -974,6 +977,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue SplitVecOp_VECREDUCE_SEQ_FDOT(SDNode *N);
SDValue SplitVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo);
SDValue SplitVecOp_UnaryOp(SDNode *N);
@@ -1121,6 +1125,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecOp_IS_FPCLASS(SDNode *N);
SDValue WidenVecOp_VECREDUCE(SDNode *N);
SDValue WidenVecOp_VECREDUCE_SEQ(SDNode *N);
+ SDValue WidenVecOp_VECREDUCE_SEQ_FDOT(SDNode *N);
SDValue WidenVecOp_VECREDUCE_FDOT(SDNode *N);
SDValue WidenVecOp_VP_REDUCE(SDNode *N);
SDValue WidenVecOp_ExpOp(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7c42ef2a9861f..7d3b6c5d949e2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -523,6 +523,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::VECREDUCE_FMIN:
case ISD::VECREDUCE_FMINIMUM:
case ISD::VECREDUCE_FMUL:
+ case ISD::VECREDUCE_FDOT: // 2-operand unordered: action from getOperand(0)
case ISD::CTTZ_ELTS:
case ISD::CTTZ_ELTS_ZERO_POISON:
case ISD::VECTOR_FIND_LAST_ACTIVE:
@@ -531,7 +532,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
- case ISD::VECREDUCE_FDOT:
+ case ISD::VECREDUCE_SEQ_FDOT: // 3-operand sequential: action from getOperand(1)
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
@@ -1317,9 +1318,43 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
- case ISD::VECREDUCE_FDOT:
- Results.push_back(TLI.expandVecReduceDot(Node, DAG));
+ case ISD::VECREDUCE_SEQ_FDOT: {
+ // Inline sequential expansion (cannot use expandVecReduceSeq — no base opcode).
+ SDLoc dl(Node);
+ SDValue AccOp = Node->getOperand(0);
+ SDValue VecA = Node->getOperand(1), VecB = Node->getOperand(2);
+ SDNodeFlags Flags = Node->getFlags();
+ EVT VT = VecA.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ if (VT.isScalableVector())
+ report_fatal_error("Expanding reductions for scalable vectors is undefined.");
+ unsigned NumElts = VT.getVectorNumElements();
+ SmallVector<SDValue, 8> OpsA, OpsB;
+ DAG.ExtractVectorElements(VecA, OpsA, 0, NumElts);
+ DAG.ExtractVectorElements(VecB, OpsB, 0, NumElts);
+ SDValue Res = AccOp;
+ if (Flags.hasAllowContract())
+ for (unsigned i = 0; i < NumElts; i++)
+ Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
+ else
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
+ Results.push_back(Res);
return;
+ }
+ case ISD::VECREDUCE_FDOT: {
+ // Decompose to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
+ SDLoc dl(Node);
+ SDValue VecA = Node->getOperand(0), VecB = Node->getOperand(1);
+ SDNodeFlags Flags = Node->getFlags();
+ EVT VT = VecA.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+ SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecA, VecB, Flags);
+ Results.push_back(DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
+ return;
+ }
case ISD::SREM:
case ISD::UREM:
ExpandREM(Node, Results);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 046caa50e0248..6505627659c2f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -933,6 +933,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_SEQ_FDOT:
+ Res = ScalarizeVecOp_VECREDUCE_SEQ_FDOT(N);
+ break;
case ISD::VECREDUCE_FDOT:
Res = ScalarizeVecOp_VECREDUCE_FDOT(N);
break;
@@ -1244,7 +1247,8 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
AccOp, Op, N->getFlags());
}
-SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N) {
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
+ // 3-operand sequential: ScalarizeVecOp_VECREDUCE_SEQ_FDOT(acc, a, b) -> FMA or FMUL+FADD.
SDLoc dl(N);
SDValue AccOp = N->getOperand(0);
SDValue VecAOp = N->getOperand(1);
@@ -1262,6 +1266,20 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N) {
return DAG.getNode(ISD::FADD, dl, EltVT, AccOp, Mul, Flags);
}
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N) {
+ // 2-operand unordered: ScalarizeVecOp_VECREDUCE_FDOT(a, b) -> FMUL(a[0], b[0]).
+ SDLoc dl(N);
+ SDValue VecAOp = N->getOperand(0);
+ SDValue VecBOp = N->getOperand(1);
+ SDNodeFlags Flags = N->getFlags();
+ EVT EltVT = N->getValueType(0);
+
+ SDValue A = GetScalarizedVector(VecAOp);
+ SDValue B = GetScalarizedVector(VecBOp);
+
+ return DAG.getNode(ISD::FMUL, dl, EltVT, A, B, Flags);
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(0));
SDValue RHS = GetScalarizedVector(N->getOperand(1));
@@ -3838,6 +3856,9 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = SplitVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_SEQ_FDOT:
+ Res = SplitVecOp_VECREDUCE_SEQ_FDOT(N);
+ break;
case ISD::VECREDUCE_FDOT:
Res = SplitVecOp_VECREDUCE_FDOT(N);
break;
@@ -4015,10 +4036,10 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
}
-SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_FDOT(SDNode *N) {
+SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
+ // 3-operand sequential: chain acc through lower half, then upper half.
EVT ResVT = N->getValueType(0);
SDLoc dl(N);
-
SDValue AccOp = N->getOperand(0);
SDValue VecAOp = N->getOperand(1);
SDValue VecBOp = N->getOperand(2);
@@ -4029,11 +4050,29 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_FDOT(SDNode *N) {
GetSplitVector(VecAOp, LoA, HiA);
GetSplitVector(VecBOp, LoB, HiB);
- // Reduce the lower half first, using the original accumulator.
+ // Reduce the lower half first using the original accumulator.
SDValue Partial =
- DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, AccOp, LoA, LoB, Flags);
- // Reduce the upper half, using the lower-half result as the new accumulator.
- return DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, Partial, HiA, HiB, Flags);
+ DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, ResVT, AccOp, LoA, LoB, Flags);
+ // Reduce the upper half with the lower-half result as the new accumulator.
+ return DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, ResVT, Partial, HiA, HiB, Flags);
+}
+
+SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_FDOT(SDNode *N) {
+ // 2-operand unordered: reduce each half independently then sum.
+ EVT ResVT = N->getValueType(0);
+ SDLoc dl(N);
+ SDValue VecAOp = N->getOperand(0);
+ SDValue VecBOp = N->getOperand(1);
+ SDNodeFlags Flags = N->getFlags();
+
+ assert(VecAOp.getValueType().isVector() && "Expected vector operand");
+ SDValue LoA, HiA, LoB, HiB;
+ GetSplitVector(VecAOp, LoA, HiA);
+ GetSplitVector(VecBOp, LoB, HiB);
+
+ SDValue Lo = DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, LoA, LoB, Flags);
+ SDValue Hi = DAG.getNode(ISD::VECREDUCE_FDOT, dl, ResVT, HiA, HiB, Flags);
+ return DAG.getNode(ISD::FADD, dl, ResVT, Lo, Hi, Flags);
}
SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
@@ -7518,6 +7557,9 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VECREDUCE_SEQ_FMUL:
Res = WidenVecOp_VECREDUCE_SEQ(N);
break;
+ case ISD::VECREDUCE_SEQ_FDOT:
+ Res = WidenVecOp_VECREDUCE_SEQ_FDOT(N);
+ break;
case ISD::VECREDUCE_FDOT:
Res = WidenVecOp_VECREDUCE_FDOT(N);
break;
@@ -8458,7 +8500,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
return DAG.getNode(Opc, dl, VT, AccOp, Op, Flags);
}
-SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_FDOT(SDNode *N) {
+SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
+ // 3-operand sequential: pad extra lanes with 0.0, re-emit SEQ_FDOT.
SDLoc dl(N);
SDValue AccOp = N->getOperand(0);
SDValue VecAOp = N->getOperand(1);
@@ -8473,19 +8516,42 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_FDOT(SDNode *N) {
EVT ElemVT = OrigVT.getVectorElementType();
SDNodeFlags Flags = N->getFlags();
- // Pad the extra lanes of both vectors with 0.0.
- // FMA(0, 0, acc) = acc, so padding with zeros is correct.
+ // Padding with 0.0 is correct: FMA(0,0,acc)=acc, FMUL(0,0)+FADD(acc,0)=acc.
SDValue Zero = DAG.getConstantFP(0.0, dl, ElemVT);
-
unsigned OrigElts = OrigVT.getVectorMinNumElements();
unsigned WideElts = WideVT.getVectorMinNumElements();
+ for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) {
+ OpA = DAG.getInsertVectorElt(dl, OpA, Zero, Idx);
+ OpB = DAG.getInsertVectorElt(dl, OpB, Zero, Idx);
+ }
+
+ return DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, VT, AccOp, OpA, OpB, Flags);
+}
+
+SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_FDOT(SDNode *N) {
+ // 2-operand unordered: pad extra lanes with 0.0, re-emit FDOT.
+ SDLoc dl(N);
+ SDValue VecAOp = N->getOperand(0);
+ SDValue VecBOp = N->getOperand(1);
+ SDValue OpA = GetWidenedVector(VecAOp);
+ SDValue OpB = GetWidenedVector(VecBOp);
+
+ EVT VT = N->getValueType(0);
+ EVT OrigVT = VecAOp.getValueType();
+ EVT WideVT = OpA.getValueType();
+ EVT ElemVT = OrigVT.getVectorElementType();
+ SDNodeFlags Flags = N->getFlags();
+
+ SDValue Zero = DAG.getConstantFP(0.0, dl, ElemVT);
+ unsigned OrigElts = OrigVT.getVectorMinNumElements();
+ unsigned WideElts = WideVT.getVectorMinNumElements();
for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) {
OpA = DAG.getInsertVectorElt(dl, OpA, Zero, Idx);
OpB = DAG.getInsertVectorElt(dl, OpB, Zero, Idx);
}
- return DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, AccOp, OpA, OpB, Flags);
+ return DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, OpA, OpB, Flags);
}
SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index eb3ec8aa8495e..50080e7bb9292 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -11210,9 +11210,14 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
break;
case Intrinsic::vector_reduce_fdot:
- // Always emit VECREDUCE_FDOT; all FMF flags (contract, reassoc, etc.)
- // are carried in SDFlags and checked during expansion.
- Res = DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, {Op1, Op2, Op3}, SDFlags);
+ if (SDFlags.hasAllowReassociation())
+ // Non-sequential: acc added externally, dot product is unordered.
+ Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
+ DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, Op2, Op3, SDFlags),
+ SDFlags);
+ else
+ // Sequential: VECREDUCE_SEQ_FDOT(acc, vecA, vecB).
+ Res = DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, VT, {Op1, Op2, Op3}, SDFlags);
break;
default:
llvm_unreachable("Unhandled vector reduce intrinsic");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 7aad3023bf484..0038241d35b56 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -568,6 +568,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::VECREDUCE_SEQ_FADD: return "vecreduce_seq_fadd";
case ISD::VECREDUCE_FMUL: return "vecreduce_fmul";
case ISD::VECREDUCE_SEQ_FMUL: return "vecreduce_seq_fmul";
+ case ISD::VECREDUCE_SEQ_FDOT: return "vecreduce_seq_fdot";
case ISD::VECREDUCE_FDOT: return "vecreduce_fdot";
case ISD::VECREDUCE_ADD: return "vecreduce_add";
case ISD::VECREDUCE_MUL: return "vecreduce_mul";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ff6d259528cac..e6aa222425d13 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12259,43 +12259,6 @@ SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) cons
return Res;
}
-SDValue TargetLowering::expandVecReduceDot(SDNode *Node,
- SelectionDAG &DAG) const {
- SDLoc dl(Node);
- SDValue AccOp = Node->getOperand(0);
- SDValue VecAOp = Node->getOperand(1);
- SDValue VecBOp = Node->getOperand(2);
- SDNodeFlags Flags = Node->getFlags();
-
- EVT VT = VecAOp.getValueType();
- EVT EltVT = VT.getVectorElementType();
-
- if (VT.isScalableVector())
- report_fatal_error(
- "Expanding reductions for scalable vectors is undefined.");
-
- unsigned NumElts = VT.getVectorNumElements();
-
- SmallVector<SDValue, 8> OpsA, OpsB;
- DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
- DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
-
- SDValue Res = AccOp;
- if (Flags.hasAllowContract()) {
- // With 'contract': use FMA chain for single-rounded dot product.
- for (unsigned i = 0; i < NumElts; i++)
- Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
- } else {
- // Default: sequential fmul + fadd (two roundings per element).
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
- Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
- }
- }
-
- return Res;
-}
-
bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index daa4b7e22dc67..bbbf9a827dfd0 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1222,7 +1222,7 @@ void TargetLoweringBase::initActions() {
ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN, ISD::VECREDUCE_FMAX,
ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM,
ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_SEQ_FMUL,
- ISD::VECREDUCE_FDOT},
+ ISD::VECREDUCE_SEQ_FDOT, ISD::VECREDUCE_FDOT},
VT, Expand);
// Named vector shuffles default to expand.
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
index 5cf90cdc2c56c..6b1e9e5355d64 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
@@ -69,6 +69,27 @@ define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
+define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; O0-LABEL: fdot_f32_reassoc:
+; O0: // %bb.0:
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: faddp v1.4s, v1.4s, v1.4s
+; O0-NEXT: // kill: def $d1 killed $d1 killed $q1
+; O0-NEXT: faddp s1, v1.2s
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f32_reassoc:
+; O1: // %bb.0:
+; O1-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O1-NEXT: faddp v1.4s, v1.4s, v1.4s
+; O1-NEXT: faddp s1, v1.2s
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: ret
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
; O0-LABEL: fdot_f16:
; O0: // %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
index 64964ab8c37ac..114b30f343ad7 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
@@ -34,3 +34,21 @@ define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
%res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+; With reassoc: tree-reduction v_mul_f32 + v_add_f32 (unordered).
+define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; GFX9-LABEL: fdot_f32_reassoc:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v2, v6
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v7
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_f32_e32 v1, v4, v8
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
index e761911126b59..d70b8b116f457 100644
--- a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
+++ b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
@@ -29,6 +29,22 @@ define float @fdot_ordered(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
+; With reassoc flag: vector fmul then shuffle-tree fadd, then add accumulator.
+define float @fdot_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @fdot_reassoc(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc <4 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX2:%.*]] = fadd reassoc <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = fadd reassoc float [[ACC:%.*]], [[TMP2]]
+; CHECK-NEXT: ret float [[BIN_RDX3]]
+;
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
; With contract flag: sequential FMA chain.
define float @fdot_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: @fdot_contract(
diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
index 87787ba2bf81c..437bf500a5de2 100644
--- a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -2568,3 +2568,167 @@ define i32 @reduce_xor_i32_nonpow2(<7 x i32> %in) {
%res = call i32 @llvm.vector.reduce.xor(<7 x i32> %in)
ret i32 %res
}
+
+declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
+
+; Default: sequential mul + add chain.
+define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-SM80-LABEL: fdot_f32(
+; CHECK-SM80: {
+; CHECK-SM80-NEXT: .reg .b32 %r<18>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT: // %bb.0:
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_param_2];
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_param_1];
+; CHECK-SM80-NEXT: ld.param.b32 %r1, [fdot_f32_param_0];
+; CHECK-SM80-NEXT: mul.rn.f32 %r10, %r3, %r7;
+; CHECK-SM80-NEXT: mul.rn.f32 %r11, %r2, %r6;
+; CHECK-SM80-NEXT: add.rn.f32 %r12, %r1, %r11;
+; CHECK-SM80-NEXT: add.rn.f32 %r13, %r12, %r10;
+; CHECK-SM80-NEXT: mul.rn.f32 %r14, %r4, %r8;
+; CHECK-SM80-NEXT: add.rn.f32 %r15, %r13, %r14;
+; CHECK-SM80-NEXT: mul.rn.f32 %r16, %r5, %r9;
+; CHECK-SM80-NEXT: add.rn.f32 %r17, %r15, %r16;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-SM80-NEXT: ret;
+;
+; CHECK-SM100-LABEL: fdot_f32(
+; CHECK-SM100: {
+; CHECK-SM100-NEXT: .reg .b32 %r<18>;
+; CHECK-SM100-NEXT: .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT: // %bb.0:
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fdot_f32_param_2];
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fdot_f32_param_1];
+; CHECK-SM100-NEXT: ld.param.b32 %r1, [fdot_f32_param_0];
+; CHECK-SM100-NEXT: mov.b64 {%r2, %r3}, %rd1;
+; CHECK-SM100-NEXT: mov.b64 {%r4, %r5}, %rd3;
+; CHECK-SM100-NEXT: mul.rn.f32 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: mul.rn.f32 %r7, %r2, %r4;
+; CHECK-SM100-NEXT: add.rn.f32 %r8, %r1, %r7;
+; CHECK-SM100-NEXT: add.rn.f32 %r9, %r8, %r6;
+; CHECK-SM100-NEXT: mov.b64 {%r10, %r11}, %rd2;
+; CHECK-SM100-NEXT: mov.b64 {%r12, %r13}, %rd4;
+; CHECK-SM100-NEXT: mul.rn.f32 %r14, %r10, %r12;
+; CHECK-SM100-NEXT: add.rn.f32 %r15, %r9, %r14;
+; CHECK-SM100-NEXT: mul.rn.f32 %r16, %r11, %r13;
+; CHECK-SM100-NEXT: add.rn.f32 %r17, %r15, %r16;
+; CHECK-SM100-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-SM100-NEXT: ret;
+ %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With contract: sequential fma chain.
+define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-SM80-LABEL: fdot_f32_contract(
+; CHECK-SM80: {
+; CHECK-SM80-NEXT: .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT: // %bb.0:
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_contract_param_2];
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_contract_param_1];
+; CHECK-SM80-NEXT: ld.param.b32 %r1, [fdot_f32_contract_param_0];
+; CHECK-SM80-NEXT: fma.rn.f32 %r10, %r2, %r6, %r1;
+; CHECK-SM80-NEXT: fma.rn.f32 %r11, %r3, %r7, %r10;
+; CHECK-SM80-NEXT: fma.rn.f32 %r12, %r4, %r8, %r11;
+; CHECK-SM80-NEXT: fma.rn.f32 %r13, %r5, %r9, %r12;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT: ret;
+;
+; CHECK-SM100-LABEL: fdot_f32_contract(
+; CHECK-SM100: {
+; CHECK-SM100-NEXT: .reg .b32 %r<14>;
+; CHECK-SM100-NEXT: .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT: // %bb.0:
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fdot_f32_contract_param_2];
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fdot_f32_contract_param_1];
+; CHECK-SM100-NEXT: ld.param.b32 %r1, [fdot_f32_contract_param_0];
+; CHECK-SM100-NEXT: mov.b64 {%r2, %r3}, %rd1;
+; CHECK-SM100-NEXT: mov.b64 {%r4, %r5}, %rd3;
+; CHECK-SM100-NEXT: fma.rn.f32 %r6, %r2, %r4, %r1;
+; CHECK-SM100-NEXT: fma.rn.f32 %r7, %r3, %r5, %r6;
+; CHECK-SM100-NEXT: mov.b64 {%r8, %r9}, %rd2;
+; CHECK-SM100-NEXT: mov.b64 {%r10, %r11}, %rd4;
+; CHECK-SM100-NEXT: fma.rn.f32 %r12, %r8, %r10, %r7;
+; CHECK-SM100-NEXT: fma.rn.f32 %r13, %r9, %r11, %r12;
+; CHECK-SM100-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-SM100-NEXT: ret;
+ %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; With reassoc: tree-reduction mul + add chain (unordered).
+define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-SM80-LABEL: fdot_f32_reassoc(
+; CHECK-SM80: {
+; CHECK-SM80-NEXT: .reg .b32 %r<18>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT: // %bb.0:
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_reassoc_param_2];
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_reassoc_param_1];
+; CHECK-SM80-NEXT: ld.param.b32 %r1, [fdot_f32_reassoc_param_0];
+; CHECK-SM80-NEXT: mul.rn.f32 %r10, %r5, %r9;
+; CHECK-SM80-NEXT: mul.rn.f32 %r11, %r4, %r8;
+; CHECK-SM80-NEXT: add.rn.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT: mul.rn.f32 %r13, %r3, %r7;
+; CHECK-SM80-NEXT: mul.rn.f32 %r14, %r2, %r6;
+; CHECK-SM80-NEXT: add.rn.f32 %r15, %r14, %r13;
+; CHECK-SM80-NEXT: add.rn.f32 %r16, %r15, %r12;
+; CHECK-SM80-NEXT: add.rn.f32 %r17, %r1, %r16;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-SM80-NEXT: ret;
+;
+; CHECK-SM100-LABEL: fdot_f32_reassoc(
+; CHECK-SM100: {
+; CHECK-SM100-NEXT: .reg .b32 %r<10>;
+; CHECK-SM100-NEXT: .reg .b64 %rd<7>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT: // %bb.0:
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fdot_f32_reassoc_param_2];
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fdot_f32_reassoc_param_1];
+; CHECK-SM100-NEXT: ld.param.b32 %r1, [fdot_f32_reassoc_param_0];
+; CHECK-SM100-NEXT: mul.rn.f32x2 %rd5, %rd2, %rd4;
+; CHECK-SM100-NEXT: mov.b64 {%r2, %r3}, %rd5;
+; CHECK-SM100-NEXT: add.rn.f32 %r4, %r2, %r3;
+; CHECK-SM100-NEXT: mul.rn.f32x2 %rd6, %rd1, %rd3;
+; CHECK-SM100-NEXT: mov.b64 {%r5, %r6}, %rd6;
+; CHECK-SM100-NEXT: add.rn.f32 %r7, %r5, %r6;
+; CHECK-SM100-NEXT: add.rn.f32 %r8, %r7, %r4;
+; CHECK-SM100-NEXT: add.rn.f32 %r9, %r1, %r8;
+; CHECK-SM100-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM100-NEXT: ret;
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; f16 variant: sequential mul + add chain.
+define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<18>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_param_1];
+; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_param_0];
+; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r1;
+; CHECK-NEXT: mov.b32 {%rs4, %rs5}, %r3;
+; CHECK-NEXT: mul.rn.f16 %rs6, %rs3, %rs5;
+; CHECK-NEXT: mul.rn.f16 %rs7, %rs2, %rs4;
+; CHECK-NEXT: add.rn.f16 %rs8, %rs1, %rs7;
+; CHECK-NEXT: add.rn.f16 %rs9, %rs8, %rs6;
+; CHECK-NEXT: mov.b32 {%rs10, %rs11}, %r2;
+; CHECK-NEXT: mov.b32 {%rs12, %rs13}, %r4;
+; CHECK-NEXT: mul.rn.f16 %rs14, %rs10, %rs12;
+; CHECK-NEXT: add.rn.f16 %rs15, %rs9, %rs14;
+; CHECK-NEXT: mul.rn.f16 %rs16, %rs11, %rs13;
+; CHECK-NEXT: add.rn.f16 %rs17, %rs15, %rs16;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs17;
+; CHECK-NEXT: ret;
+ %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll b/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
deleted file mode 100644
index 90deab32f7759..0000000000000
--- a/llvm/test/CodeGen/NVPTX/vecreduce-fdot.ll
+++ /dev/null
@@ -1,82 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
-; RUN: -disable-post-ra -verify-machineinstrs \
-; RUN: | FileCheck %s
-target triple = "nvptx64-nvidia-cuda"
-target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-
-declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
-declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
-
-; Default: sequential mul + add chain.
-define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fdot_f32(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<18>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_param_2];
-; CHECK-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_param_1];
-; CHECK-NEXT: ld.param.b32 %r1, [fdot_f32_param_0];
-; CHECK-NEXT: mul.rn.f32 %r10, %r3, %r7;
-; CHECK-NEXT: mul.rn.f32 %r11, %r2, %r6;
-; CHECK-NEXT: add.rn.f32 %r12, %r1, %r11;
-; CHECK-NEXT: add.rn.f32 %r13, %r12, %r10;
-; CHECK-NEXT: mul.rn.f32 %r14, %r4, %r8;
-; CHECK-NEXT: add.rn.f32 %r15, %r13, %r14;
-; CHECK-NEXT: mul.rn.f32 %r16, %r5, %r9;
-; CHECK-NEXT: add.rn.f32 %r17, %r15, %r16;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r17;
-; CHECK-NEXT: ret;
- %res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
- ret float %res
-}
-
-; With contract: sequential fma chain.
-define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fdot_f32_contract(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<14>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_contract_param_2];
-; CHECK-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_contract_param_1];
-; CHECK-NEXT: ld.param.b32 %r1, [fdot_f32_contract_param_0];
-; CHECK-NEXT: fma.rn.f32 %r10, %r2, %r6, %r1;
-; CHECK-NEXT: fma.rn.f32 %r11, %r3, %r7, %r10;
-; CHECK-NEXT: fma.rn.f32 %r12, %r4, %r8, %r11;
-; CHECK-NEXT: fma.rn.f32 %r13, %r5, %r9, %r12;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
-; CHECK-NEXT: ret;
- %res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
- ret float %res
-}
-
-; f16 variant: sequential mul + add chain.
-define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: fdot_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<18>;
-; CHECK-NEXT: .reg .b32 %r<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_param_2];
-; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_param_1];
-; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_param_0];
-; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r1;
-; CHECK-NEXT: mov.b32 {%rs4, %rs5}, %r3;
-; CHECK-NEXT: mul.rn.f16 %rs6, %rs3, %rs5;
-; CHECK-NEXT: mul.rn.f16 %rs7, %rs2, %rs4;
-; CHECK-NEXT: add.rn.f16 %rs8, %rs1, %rs7;
-; CHECK-NEXT: add.rn.f16 %rs9, %rs8, %rs6;
-; CHECK-NEXT: mov.b32 {%rs10, %rs11}, %r2;
-; CHECK-NEXT: mov.b32 {%rs12, %rs13}, %r4;
-; CHECK-NEXT: mul.rn.f16 %rs14, %rs10, %rs12;
-; CHECK-NEXT: add.rn.f16 %rs15, %rs9, %rs14;
-; CHECK-NEXT: mul.rn.f16 %rs16, %rs11, %rs13;
-; CHECK-NEXT: add.rn.f16 %rs17, %rs15, %rs16;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs17;
-; CHECK-NEXT: ret;
- %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
- ret half %res
-}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
index 90eeb1a5eee31..f49efb1cb18ff 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
@@ -29,3 +29,17 @@ define arm_aapcs_vfpcc float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x
%res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+; With reassoc: tree-reduction vmul.f32 + vadd.f32 (unordered).
+define arm_aapcs_vfpcc float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32_reassoc:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vmul.f32 q1, q1, q2
+; CHECK-NEXT: vadd.f32 s2, s6, s7
+; CHECK-NEXT: vadd.f32 s4, s4, s5
+; CHECK-NEXT: vadd.f32 s2, s4, s2
+; CHECK-NEXT: vadd.f32 s0, s0, s2
+; CHECK-NEXT: bx lr
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
index f40945bdb6382..92ca5eef4b43a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
@@ -45,6 +45,47 @@ define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
+; With reassoc: pairwise vmulss + vaddss tree (unordered).
+define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
+; AVX2-LABEL: fdot_f32_reassoc:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX2-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; FMA-LABEL: fdot_f32_reassoc:
+; FMA: # %bb.0:
+; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; FMA-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; FMA-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
+; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; FMA-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; FMA-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; FMA-NEXT: retq
+ %res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
; With contract and FMA hardware: vfmadd chain.
define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; AVX2-LABEL: fdot_f32_contract:
>From 567afc43111145f82568e57f9f115c506c6471fe Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 11:35:48 -0700
Subject: [PATCH 03/17] Clarify contract semantics in fdot GlobalISel lowering
The 'contract' flag means "permission to fuse" (not "must fuse"), so
emitting G_FMUL + G_VECREDUCE_SEQ_FADD with propagated flags is
semantically correct. Add a comment documenting that contract-to-FMA
fusion is deferred to combine passes, and that targets with dedicated
dot-product instructions should select directly from the
G_FMUL + G_VECREDUCE_SEQ_FADD pattern.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 76a8ae38daaa8..68f9b3265ef14 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2572,8 +2572,15 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
return true;
}
case Intrinsic::vector_reduce_fdot: {
- // Lower as G_FMUL(vecA, vecB) followed by G_VECREDUCE_SEQ_FADD or
- // G_VECREDUCE_FADD + G_FADD depending on the reassoc flag.
+ // Lower fdot as a vector fmul followed by a reduction:
+ // no reassoc: G_FMUL(vecA, vecB) -> G_VECREDUCE_SEQ_FADD(acc, products)
+ // reassoc: G_FMUL(vecA, vecB) -> G_FADD(acc, G_VECREDUCE_FADD(products))
+ //
+ // Note: when the 'contract' flag is set, the MIFlags are propagated to
+ // both G_FMUL and G_VECREDUCE_SEQ_FADD, permitting later combine passes
+ // to fuse adjacent fmul+fadd pairs into FMA instructions. This is not
+ // guaranteed — targets with dedicated dot-product instructions should
+ // select directly from this pattern without relying on FMA fusion.
Register Dst = getOrCreateVReg(CI);
Register AccSrc = getOrCreateVReg(*CI.getArgOperand(0));
Register VecA = getOrCreateVReg(*CI.getArgOperand(1));
>From 793cfbb429013a03cdd32e1fa1d40dc775ac5f73 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 11:52:02 -0700
Subject: [PATCH 04/17] Place VECREDUCE_SEQ_FDOT and VECREDUCE_FDOT adjacent in
ISDOpcodes.h
Move VECREDUCE_SEQ_FDOT from the SEQ_FADD/SEQ_FMUL group to immediately
before VECREDUCE_FDOT (after VECREDUCE_FMINIMUM), so the two fdot variants
are grouped together.
Regenerate X86 and AMDGPU fdot test CHECK lines: the opcode reorder
changed the tree-reduction code path for the reassoc case, now correctly
emitting a vector vmulps + pairwise shuffle-tree instead of scalar chains.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 12 +++----
.../test/CodeGen/AMDGPU/vector-reduce-fdot.ll | 12 +++----
llvm/test/CodeGen/X86/vector-reduce-fdot.ll | 36 ++++++-------------
3 files changed, 22 insertions(+), 38 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 33e14e739014a..2a92ce9a5b9a8 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1487,12 +1487,6 @@ enum NodeType {
/// RES = (((ACC + SRC_VEC[0]) + SRC_VEC[1]) + SRC_VEC[2]) + SRC_VEC[3]
VECREDUCE_SEQ_FADD,
VECREDUCE_SEQ_FMUL,
- /// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
- /// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
- /// left-to-right order.
- /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
- /// With 'contract' flag: sequential FMA chain (single rounding per element).
- VECREDUCE_SEQ_FDOT,
/// These reductions have relaxed evaluation order semantics, and have a
/// single vector operand. The order of evaluation is unspecified. For
@@ -1515,6 +1509,12 @@ enum NodeType {
/// llvm.minimum and llvm.maximum semantics.
VECREDUCE_FMAXIMUM,
VECREDUCE_FMINIMUM,
+ /// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
+ /// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
+ /// left-to-right order.
+ /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
+ /// With 'contract' flag: sequential FMA chain (single rounding per element).
+ VECREDUCE_SEQ_FDOT,
/// VECREDUCE_FDOT(VecA, VecB) - Unordered floating-point dot product
/// reduction. Computes sum(VecA[i] * VecB[i]) with unspecified evaluation
/// order. The caller adds the accumulator via a separate FADD. Decomposes
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
index 114b30f343ad7..11ef7e81ccfe8 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
@@ -40,13 +40,13 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
; GFX9-LABEL: fdot_f32_reassoc:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v7
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, v2, v6
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, v3, v7
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, v4, v8
+; GFX9-NEXT: v_mul_f32_e32 v4, v4, v8
+; GFX9-NEXT: v_mul_f32_e32 v2, v2, v6
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v2
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
%res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
index 92ca5eef4b43a..991a287ac97a6 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
@@ -49,37 +49,21 @@ define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
; AVX2-LABEL: fdot_f32_reassoc:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm3
-; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; AVX2-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
-; AVX2-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
-; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmulps %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX2-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vaddss %xmm2, %xmm1, %xmm1
; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; FMA-LABEL: fdot_f32_reassoc:
; FMA: # %bb.0:
-; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm3
-; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; FMA-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; FMA-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
-; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; FMA-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
-; FMA-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
-; FMA-NEXT: vmulss %xmm4, %xmm3, %xmm3
-; FMA-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; FMA-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; FMA-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; FMA-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vmulps %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; FMA-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; FMA-NEXT: vaddss %xmm2, %xmm1, %xmm1
; FMA-NEXT: vaddss %xmm1, %xmm0, %xmm0
; FMA-NEXT: retq
%res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
>From 1245df7ea978e1ec8a60dd1eba38fac92d1c59b8 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 12:08:22 -0700
Subject: [PATCH 05/17] Add G_VECREDUCE_SEQ_FDOT/G_VECREDUCE_FDOT GlobalISel
opcodes; separate ISD groups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- Separate VECREDUCE_SEQ_FDOT and VECREDUCE_FDOT in ISDOpcodes.h to mirror the
GenericOpcodes.td structure: SEQ_FDOT in the sequential group (after SEQ_FMUL),
FDOT in the unordered group (after FMINIMUM).
- Add dedicated GlobalISel opcodes G_VECREDUCE_SEQ_FDOT (4-operand: dst, acc, va, vb)
and G_VECREDUCE_FDOT (3-operand: dst, va, vb) to GenericOpcodes.td and
TargetOpcodes.def.
- Update IRTranslator: emit G_VECREDUCE_SEQ_FDOT (no reassoc) or
G_FADD(acc, G_VECREDUCE_FDOT(va, vb)) (reassoc) instead of decomposing to
G_FMUL + G_VECREDUCE_SEQ_FADD eagerly.
- Add LegalizerHelper::lower() cases:
- G_VECREDUCE_FDOT → G_FMUL(va, vb) + G_VECREDUCE_FADD(products)
- G_VECREDUCE_SEQ_FDOT → G_FMUL(va, vb) + G_VECREDUCE_SEQ_FADD(acc, products)
- Add MachineVerifier checks for the new opcodes.
- Register G_VECREDUCE_FDOT → vecreduce_fdot in SelectionDAGCompat.td.
- Add builder helpers buildVecReduceSeqFDot / buildVecReduceFDot in MachineIRBuilder.h.
- Register G_VECREDUCE_FDOT in GVecReduce::classof() and the GISEL_VECREDUCE
case macros in Utils.h.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 1 +
.../CodeGen/GlobalISel/MachineIRBuilder.h | 21 +++++++++++++
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 2 ++
llvm/include/llvm/CodeGen/ISDOpcodes.h | 12 +++----
llvm/include/llvm/Support/TargetOpcodes.def | 2 ++
llvm/include/llvm/Target/GenericOpcodes.td | 18 +++++++++++
.../Target/GlobalISel/SelectionDAGCompat.td | 1 +
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 22 ++++---------
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 29 +++++++++++++++++
llvm/lib/CodeGen/MachineVerifier.cpp | 31 +++++++++++++++++++
10 files changed, 117 insertions(+), 22 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 5faf57fd06228..c0e584ff0cb0e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -572,6 +572,7 @@ class GVecReduce : public GenericMachineInstr {
public:
static bool classof(const MachineInstr *MI) {
switch (MI->getOpcode()) {
+ case TargetOpcode::G_VECREDUCE_FDOT:
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_FMUL:
case TargetOpcode::G_VECREDUCE_FMAX:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index bc80a8b756464..6942a7dec2344 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -2311,6 +2311,27 @@ class LLVM_ABI MachineIRBuilder {
{ScalarIn, {VecIn}});
}
+ /// Build and insert \p Res = G_VECREDUCE_SEQ_FDOT \p Acc, \p VecA, \p VecB
+ ///
+ /// \p Acc is the scalar accumulator. Computes Acc + sum(VecA[i] * VecB[i])
+ /// in strict left-to-right order.
+ MachineInstrBuilder buildVecReduceSeqFDot(const DstOp &Dst,
+ const SrcOp &Acc,
+ const SrcOp &VecA,
+ const SrcOp &VecB) {
+ return buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FDOT, {Dst},
+ {Acc, VecA, VecB});
+ }
+
+ /// Build and insert \p Res = G_VECREDUCE_FDOT \p VecA, \p VecB
+ ///
+ /// Computes sum(VecA[i] * VecB[i]) in unspecified order (no accumulator).
+ MachineInstrBuilder buildVecReduceFDot(const DstOp &Dst,
+ const SrcOp &VecA,
+ const SrcOp &VecB) {
+ return buildInstr(TargetOpcode::G_VECREDUCE_FDOT, {Dst}, {VecA, VecB});
+ }
+
/// Build and insert \p Res = G_VECREDUCE_FADD \p Src
///
/// \p ScalarIn is the scalar accumulator input to the reduction operation of
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 84b4e84f0c64d..ec73dcdabe6ee 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -56,6 +56,7 @@ class APFloat;
#define GISEL_VECREDUCE_CASES_ALL \
case TargetOpcode::G_VECREDUCE_SEQ_FADD: \
case TargetOpcode::G_VECREDUCE_SEQ_FMUL: \
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT: \
case TargetOpcode::G_VECREDUCE_FADD: \
case TargetOpcode::G_VECREDUCE_FMUL: \
case TargetOpcode::G_VECREDUCE_FMAX: \
@@ -73,6 +74,7 @@ class APFloat;
case TargetOpcode::G_VECREDUCE_UMIN:
#define GISEL_VECREDUCE_CASES_NONSEQ \
+ case TargetOpcode::G_VECREDUCE_FDOT: \
case TargetOpcode::G_VECREDUCE_FADD: \
case TargetOpcode::G_VECREDUCE_FMUL: \
case TargetOpcode::G_VECREDUCE_FMAX: \
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 2a92ce9a5b9a8..33e14e739014a 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1487,6 +1487,12 @@ enum NodeType {
/// RES = (((ACC + SRC_VEC[0]) + SRC_VEC[1]) + SRC_VEC[2]) + SRC_VEC[3]
VECREDUCE_SEQ_FADD,
VECREDUCE_SEQ_FMUL,
+ /// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
+ /// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
+ /// left-to-right order.
+ /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
+ /// With 'contract' flag: sequential FMA chain (single rounding per element).
+ VECREDUCE_SEQ_FDOT,
/// These reductions have relaxed evaluation order semantics, and have a
/// single vector operand. The order of evaluation is unspecified. For
@@ -1509,12 +1515,6 @@ enum NodeType {
/// llvm.minimum and llvm.maximum semantics.
VECREDUCE_FMAXIMUM,
VECREDUCE_FMINIMUM,
- /// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
- /// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
- /// left-to-right order.
- /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
- /// With 'contract' flag: sequential FMA chain (single rounding per element).
- VECREDUCE_SEQ_FDOT,
/// VECREDUCE_FDOT(VecA, VecB) - Unordered floating-point dot product
/// reduction. Computes sum(VecA[i] * VecB[i]) with unspecified evaluation
/// order. The caller adds the accumulator via a separate FADD. Decomposes
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index a100e2e3cdaf1..724f6d3668ba6 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -958,12 +958,14 @@ HANDLE_TARGET_OPCODE(G_UBSANTRAP)
/// Vector reductions
HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD)
HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL)
+HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FDOT)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FADD)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMUL)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAX)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMIN)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAXIMUM)
HANDLE_TARGET_OPCODE(G_VECREDUCE_FMINIMUM)
+HANDLE_TARGET_OPCODE(G_VECREDUCE_FDOT)
HANDLE_TARGET_OPCODE(G_VECREDUCE_ADD)
HANDLE_TARGET_OPCODE(G_VECREDUCE_MUL)
HANDLE_TARGET_OPCODE(G_VECREDUCE_AND)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 7f6d868c44006..eb8ee276f9719 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1735,6 +1735,15 @@ def G_VECREDUCE_SEQ_FMUL : GenericInstruction {
let hasSideEffects = false;
}
+// G_VECREDUCE_SEQ_FDOT(acc, vecA, vecB): Sequential dot-product reduction.
+// Computes acc + sum(vecA[i] * vecB[i]) left-to-right.
+// 'contract' flag: FMA chain. Default: fmul+fadd chain.
+def G_VECREDUCE_SEQ_FDOT : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$acc, type2:$va, type2:$vb);
+ let hasSideEffects = false;
+}
+
def G_VECREDUCE_FADD : VectorReduction;
def G_VECREDUCE_FMUL : VectorReduction;
@@ -1743,6 +1752,15 @@ def G_VECREDUCE_FMIN : VectorReduction;
def G_VECREDUCE_FMAXIMUM : VectorReduction;
def G_VECREDUCE_FMINIMUM : VectorReduction;
+// G_VECREDUCE_FDOT(vecA, vecB): Unordered dot-product reduction.
+// Computes sum(vecA[i] * vecB[i]) in unspecified order.
+// Accumulator added by caller via G_FADD. Decomposes to G_FMUL + G_VECREDUCE_FADD.
+def G_VECREDUCE_FDOT : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$va, type1:$vb);
+ let hasSideEffects = false;
+}
+
def G_VECREDUCE_ADD : VectorReduction;
def G_VECREDUCE_MUL : VectorReduction;
def G_VECREDUCE_AND : VectorReduction;
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 4b0c89b780c6f..9f6e2601328b5 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -205,6 +205,7 @@ def : GINodeEquiv<G_ROTL, rotl>;
def : GINodeEquiv<G_LROUND, lround>;
def : GINodeEquiv<G_LLROUND, llround>;
def : GINodeEquiv<G_VECREDUCE_FADD, vecreduce_fadd>;
+def : GINodeEquiv<G_VECREDUCE_FDOT, vecreduce_fdot>;
def : GINodeEquiv<G_VECREDUCE_FMAX, vecreduce_fmax>;
def : GINodeEquiv<G_VECREDUCE_FMIN, vecreduce_fmin>;
def : GINodeEquiv<G_VECREDUCE_FMAXIMUM, vecreduce_fmaximum>;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 68f9b3265ef14..43ed063f8e590 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2572,29 +2572,19 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
return true;
}
case Intrinsic::vector_reduce_fdot: {
- // Lower fdot as a vector fmul followed by a reduction:
- // no reassoc: G_FMUL(vecA, vecB) -> G_VECREDUCE_SEQ_FADD(acc, products)
- // reassoc: G_FMUL(vecA, vecB) -> G_FADD(acc, G_VECREDUCE_FADD(products))
- //
- // Note: when the 'contract' flag is set, the MIFlags are propagated to
- // both G_FMUL and G_VECREDUCE_SEQ_FADD, permitting later combine passes
- // to fuse adjacent fmul+fadd pairs into FMA instructions. This is not
- // guaranteed — targets with dedicated dot-product instructions should
- // select directly from this pattern without relying on FMA fusion.
+ // Emit dedicated dot-product reduction opcodes:
+ // no reassoc: G_VECREDUCE_SEQ_FDOT(acc, vecA, vecB) — sequential
+ // reassoc: G_FADD(acc, G_VECREDUCE_FDOT(vecA, vecB)) — unordered
Register Dst = getOrCreateVReg(CI);
Register AccSrc = getOrCreateVReg(*CI.getArgOperand(0));
Register VecA = getOrCreateVReg(*CI.getArgOperand(1));
Register VecB = getOrCreateVReg(*CI.getArgOperand(2));
- LLT VecTy = MRI->getType(VecA);
+ LLT DstTy = MRI->getType(Dst);
auto MIFlags = MachineInstr::copyFlagsFromInstruction(CI);
- auto Prod = MIRBuilder.buildFMul(VecTy, VecA, VecB, MIFlags);
if (!CI.hasAllowReassoc()) {
- MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FADD, {Dst},
- {AccSrc, Prod}, MIFlags);
+ MIRBuilder.buildVecReduceSeqFDot(Dst, AccSrc, VecA, VecB);
} else {
- LLT DstTy = MRI->getType(Dst);
- auto Rdx = MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_FADD, {DstTy},
- {Prod}, MIFlags);
+ auto Rdx = MIRBuilder.buildVecReduceFDot(DstTy, VecA, VecB);
MIRBuilder.buildFAdd(Dst, AccSrc, Rdx, MIFlags);
}
return true;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index e78dfb0565439..3c90775186f47 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4921,6 +4921,21 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
return lowerTRUNC(MI);
GISEL_VECREDUCE_CASES_NONSEQ
return lowerVectorReduction(MI);
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
+ // G_VECREDUCE_SEQ_FDOT(dst, acc, vecA, vecB) ->
+ // G_FMUL(vecA, vecB), G_VECREDUCE_SEQ_FADD(acc, products)
+ Register DstReg = MI.getOperand(0).getReg();
+ Register AccReg = MI.getOperand(1).getReg();
+ Register VecA = MI.getOperand(2).getReg();
+ Register VecB = MI.getOperand(3).getReg();
+ LLT VecTy = MRI.getType(VecA);
+ unsigned Flags = MI.getFlags();
+ auto Products = MIRBuilder.buildFMul(VecTy, VecA, VecB, Flags);
+ MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FADD, {DstReg},
+ {AccReg, Products.getReg(0)}, Flags);
+ MI.eraseFromParent();
+ return Legalized;
+ }
case G_VAARG:
return lowerVAArg(MI);
case G_ATOMICRMW_SUB: {
@@ -10469,6 +10484,20 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
+ // G_VECREDUCE_FDOT(dst, vecA, vecB) -> G_FMUL(vecA, vecB), G_VECREDUCE_FADD(products)
+ if (MI.getOpcode() == TargetOpcode::G_VECREDUCE_FDOT) {
+ Register DstReg = MI.getOperand(0).getReg();
+ Register VecA = MI.getOperand(1).getReg();
+ Register VecB = MI.getOperand(2).getReg();
+ LLT VecTy = MRI.getType(VecA);
+ unsigned Flags = MI.getFlags();
+ auto Products = MIRBuilder.buildFMul(VecTy, VecA, VecB, Flags);
+ MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_FADD, {DstReg},
+ {Products.getReg(0)}, Flags);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
Register SrcReg = MI.getOperand(1).getReg();
LLT SrcTy = MRI.getType(SrcReg);
LLT DstTy = MRI.getType(SrcReg);
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index c3be06384e317..7cd5b26c1e7fb 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2152,6 +2152,37 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
report("Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
break;
}
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
+ LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+ LLT AccTy = MRI->getType(MI->getOperand(1).getReg());
+ LLT VecATy = MRI->getType(MI->getOperand(2).getReg());
+ LLT VecBTy = MRI->getType(MI->getOperand(3).getReg());
+ if (!DstTy.isScalar())
+ report("Vector reduction requires a scalar destination type", MI);
+ if (!AccTy.isScalar())
+ report("G_VECREDUCE_SEQ_FDOT requires a scalar accumulator operand", MI);
+ if (!VecATy.isVector())
+ report("G_VECREDUCE_SEQ_FDOT requires a vector 2nd operand", MI);
+ if (!VecBTy.isVector())
+ report("G_VECREDUCE_SEQ_FDOT requires a vector 3rd operand", MI);
+ if (VecATy != VecBTy)
+ report("G_VECREDUCE_SEQ_FDOT vector operands must have the same type", MI);
+ break;
+ }
+ case TargetOpcode::G_VECREDUCE_FDOT: {
+ LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+ LLT VecATy = MRI->getType(MI->getOperand(1).getReg());
+ LLT VecBTy = MRI->getType(MI->getOperand(2).getReg());
+ if (!DstTy.isScalar())
+ report("Vector reduction requires a scalar destination type", MI);
+ if (!VecATy.isVector())
+ report("G_VECREDUCE_FDOT requires a vector 1st operand", MI);
+ if (!VecBTy.isVector())
+ report("G_VECREDUCE_FDOT requires a vector 2nd operand", MI);
+ if (VecATy != VecBTy)
+ report("G_VECREDUCE_FDOT vector operands must have the same type", MI);
+ break;
+ }
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_FMUL:
case TargetOpcode::G_VECREDUCE_FMAX:
>From 8089f514c4c1491452e3375a2e159f1c56e71c5f Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 12:20:54 -0700
Subject: [PATCH 06/17] Fix legalizer-info-validation and AArch64
vecreduce-fdot tests
Update tests broken by G_VECREDUCE_SEQ_FDOT/G_VECREDUCE_FDOT additions:
- AArch64 and RISCV legalizer-info-validation.mir: insert the two new
opcodes at their correct positions in the expected DEBUG-NEXT output
(SEQ_FDOT after SEQ_FMUL, FDOT after FMINIMUM).
- AArch64/vecreduce-fdot.ll: regenerate CHECK lines after the
ISDOpcodes.h opcode reorder.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
.../GlobalISel/legalizer-info-validation.mir | 6 +
llvm/test/CodeGen/AArch64/vecreduce-fdot.ll | 105 +++++++++++-------
.../GlobalISel/legalizer-info-validation.mir | 6 +
3 files changed, 75 insertions(+), 42 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 13b765031d614..a0e7b1fc255f5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -918,6 +918,9 @@
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_VECREDUCE_SEQ_FDOT (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_VECREDUCE_FADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
@@ -939,6 +942,9 @@
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_VECREDUCE_FDOT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_VECREDUCE_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 2, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
index 6b1e9e5355d64..4b9410ca0e627 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
@@ -8,17 +8,20 @@ declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
; O0-LABEL: fdot_f32:
; O0: // %bb.0:
-; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
-; O0-NEXT: fmov s4, s1
-; O0-NEXT: mov s3, v1.s[1]
-; O0-NEXT: mov s2, v1.s[2]
-; O0-NEXT: mov s1, v1.s[3]
+; O0-NEXT: mov v5.16b, v1.16b
+; O0-NEXT: mov s1, v5.s[3]
+; O0-NEXT: mov s3, v5.s[2]
+; O0-NEXT: mov s4, v5.s[1]
+; O0-NEXT: fmul s4, s4, v2.s[1]
+; O0-NEXT: fmov w0, s5
+; O0-NEXT: fmov s5, w0
+; O0-NEXT: fmul s5, s5, v2.s[0]
+; O0-NEXT: fadd s0, s0, s5
; O0-NEXT: fadd s0, s0, s4
+; O0-NEXT: fmul s3, s3, v2.s[2]
; O0-NEXT: fadd s0, s0, s3
-; O0-NEXT: fadd s0, s0, s2
+; O0-NEXT: fmul s1, s1, v2.s[3]
; O0-NEXT: fadd s0, s0, s1
-; O0-NEXT: fmov w0, s0
-; O0-NEXT: fmov s0, w0
; O0-NEXT: ret
;
; O1-LABEL: fdot_f32:
@@ -42,17 +45,24 @@ define float @fdot_f32(float %acc, <4 x float> %a, <4 x float> %b) {
define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; O0-LABEL: fdot_f32_contract:
; O0: // %bb.0:
-; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
-; O0-NEXT: fmov s4, s1
-; O0-NEXT: mov s3, v1.s[1]
-; O0-NEXT: mov s2, v1.s[2]
-; O0-NEXT: mov s1, v1.s[3]
-; O0-NEXT: fadd s0, s0, s4
-; O0-NEXT: fadd s0, s0, s3
-; O0-NEXT: fadd s0, s0, s2
-; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: sub sp, sp, #16
+; O0-NEXT: .cfi_def_cfa_offset 16
+; O0-NEXT: str q2, [sp] // 16-byte Spill
+; O0-NEXT: mov v2.16b, v1.16b
+; O0-NEXT: fmov s6, s0
+; O0-NEXT: ldr q0, [sp] // 16-byte Reload
+; O0-NEXT: mov s1, v0.s[3]
+; O0-NEXT: mov s3, v0.s[2]
+; O0-NEXT: mov s4, v0.s[1]
; O0-NEXT: fmov w0, s0
-; O0-NEXT: fmov s0, w0
+; O0-NEXT: fmov w1, s2
+; O0-NEXT: fmov s0, w1
+; O0-NEXT: fmov s5, w0
+; O0-NEXT: fmadd s0, s0, s5, s6
+; O0-NEXT: fmla s0, s4, v2.s[1]
+; O0-NEXT: fmla s0, s3, v2.s[2]
+; O0-NEXT: fmla s0, s1, v2.s[3]
+; O0-NEXT: add sp, sp, #16
; O0-NEXT: ret
;
; O1-LABEL: fdot_f32_contract:
@@ -74,7 +84,8 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
; O0: // %bb.0:
; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
; O0-NEXT: faddp v1.4s, v1.4s, v1.4s
-; O0-NEXT: // kill: def $d1 killed $d1 killed $q1
+; O0-NEXT: fmov x0, d1
+; O0-NEXT: fmov d1, x0
; O0-NEXT: faddp s1, v1.2s
; O0-NEXT: fadd s0, s0, s1
; O0-NEXT: ret
@@ -93,40 +104,50 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
; O0-LABEL: fdot_f16:
; O0: // %bb.0:
-; O0-NEXT: fcvtl v1.4s, v1.4h
-; O0-NEXT: fcvtl v2.4s, v2.4h
-; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
-; O0-NEXT: fcvtn v5.4h, v1.4s
-; O0-NEXT: fmov s4, s5
-; O0-NEXT: // implicit-def: $q1
-; O0-NEXT: fmov d1, d5
-; O0-NEXT: mov h3, v1.h[1]
-; O0-NEXT: // implicit-def: $q1
-; O0-NEXT: fmov d1, d5
-; O0-NEXT: mov h2, v1.h[2]
-; O0-NEXT: // implicit-def: $q1
-; O0-NEXT: fmov d1, d5
-; O0-NEXT: mov h1, v1.h[3]
+; O0-NEXT: // implicit-def: $q5
+; O0-NEXT: fmov d5, d1
+; O0-NEXT: mov h1, v5.h[3]
+; O0-NEXT: // implicit-def: $q16
+; O0-NEXT: fmov d16, d2
+; O0-NEXT: mov h2, v16.h[3]
+; O0-NEXT: mov h3, v5.h[2]
+; O0-NEXT: mov h4, v16.h[2]
+; O0-NEXT: fmov s6, s5
+; O0-NEXT: fmov s7, s16
+; O0-NEXT: mov h5, v5.h[1]
+; O0-NEXT: mov h16, v16.h[1]
+; O0-NEXT: fcvt s16, h16
+; O0-NEXT: fcvt s5, h5
+; O0-NEXT: fmul s5, s5, s16
+; O0-NEXT: fcvt h5, s5
+; O0-NEXT: fcvt s5, h5
+; O0-NEXT: fcvt s7, h7
+; O0-NEXT: fcvt s6, h6
+; O0-NEXT: fmul s6, s6, s7
+; O0-NEXT: fcvt h6, s6
+; O0-NEXT: fcvt s6, h6
; O0-NEXT: fcvt s0, h0
-; O0-NEXT: fcvt s4, h4
-; O0-NEXT: fadd s0, s0, s4
+; O0-NEXT: fadd s0, s0, s6
; O0-NEXT: fcvt h0, s0
; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fadd s0, s0, s5
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s4, h4
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fmul s3, s3, s4
+; O0-NEXT: fcvt h3, s3
; O0-NEXT: fcvt s3, h3
; O0-NEXT: fadd s0, s0, s3
; O0-NEXT: fcvt h0, s0
; O0-NEXT: fcvt s0, h0
; O0-NEXT: fcvt s2, h2
-; O0-NEXT: fadd s0, s0, s2
-; O0-NEXT: fcvt h0, s0
-; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fmul s1, s1, s2
+; O0-NEXT: fcvt h1, s1
; O0-NEXT: fcvt s1, h1
; O0-NEXT: fadd s0, s0, s1
; O0-NEXT: fcvt h0, s0
-; O0-NEXT: // kill: def $s0 killed $h0
-; O0-NEXT: fmov w0, s0
-; O0-NEXT: fmov s0, w0
-; O0-NEXT: // kill: def $h0 killed $h0 killed $s0
; O0-NEXT: ret
;
; O1-LABEL: fdot_f16:
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 8701ba3786989..c0de18c953a32 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -906,6 +906,9 @@
# DEBUG-NEXT: G_VECREDUCE_SEQ_FMUL (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_VECREDUCE_SEQ_FDOT (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_VECREDUCE_FADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -924,6 +927,9 @@
# DEBUG-NEXT: G_VECREDUCE_FMINIMUM (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_VECREDUCE_FDOT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_VECREDUCE_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
>From 4bad9e356bac96df8b96e8241a11e27955e61aa8 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 12:47:29 -0700
Subject: [PATCH 07/17] Add vecreduce_fdot SDNode and SDTFPDotVecReduce type
profile
Defines the TableGen SDNode for ISD::VECREDUCE_FDOT so it can be used
in instruction selection patterns and satisfies the GINodeEquiv reference
in SelectionDAGCompat.td.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/Target/TargetSelectionDAG.td | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 573342846b4cf..67c9570804125 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -311,6 +311,9 @@ def SDTVecReduce : SDTypeProfile<1, 1, [ // vector reduction
def SDTFPVecReduce : SDTypeProfile<1, 1, [ // FP vector reduction
SDTCisFP<0>, SDTCisEltOfVec<0, 1>
]>;
+def SDTFPDotVecReduce : SDTypeProfile<1, 2, [ // FP dot product vector reduction
+ SDTCisFP<0>, SDTCisEltOfVec<0, 1>, SDTCisSameAs<1, 2>
+]>;
def SDTVecReverse : SDTypeProfile<1, 1, [ // vector reverse
SDTCisVec<0>, SDTCisSameAs<0,1>
@@ -560,6 +563,7 @@ def vecreduce_fmin : SDNode<"ISD::VECREDUCE_FMIN", SDTFPVecReduce>;
def vecreduce_fmax : SDNode<"ISD::VECREDUCE_FMAX", SDTFPVecReduce>;
def vecreduce_fminimum : SDNode<"ISD::VECREDUCE_FMINIMUM", SDTFPVecReduce>;
def vecreduce_fmaximum : SDNode<"ISD::VECREDUCE_FMAXIMUM", SDTFPVecReduce>;
+def vecreduce_fdot : SDNode<"ISD::VECREDUCE_FDOT", SDTFPDotVecReduce>;
def partial_reduce_umla : SDNode<"ISD::PARTIAL_REDUCE_UMLA",
SDTPartialReduceMLA>;
>From b5d34f82b1532dd6d68dab07d5d79bb3d1edeb1d Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Sat, 11 Apr 2026 17:11:10 -0700
Subject: [PATCH 08/17] Extract expandVecReduceSeqDot/expandVecReduceDot into
TLI methods
Move the inline VECREDUCE_SEQ_FDOT and VECREDUCE_FDOT expansion logic from
LegalizeVectorOps.cpp into TargetLowering methods, mirroring the existing
expandVecReduceSeq/expandVecReduce pattern. The new methods are reused
across all three expansion sites: LegalizeVectorOps, SoftenFloatRes, and
SoftPromoteHalfRes.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 9 +++
.../SelectionDAG/LegalizeFloatTypes.cpp | 68 ++-----------------
.../SelectionDAG/LegalizeVectorOps.cpp | 39 ++---------
.../CodeGen/SelectionDAG/TargetLowering.cpp | 45 ++++++++++++
4 files changed, 62 insertions(+), 99 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 59a0f2d2e0c2a..55f9e6035018d 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5863,6 +5863,15 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
+ /// Expand a VECREDUCE_SEQ_FDOT into a sequential chain of fmul+fadd (or FMA
+ /// with the 'contract' flag). The accumulator is operand 0; vectors are
+ /// operands 1 and 2.
+ SDValue expandVecReduceSeqDot(SDNode *Node, SelectionDAG &DAG) const;
+
+ /// Expand a VECREDUCE_FDOT into FMUL(vecA, vecB) followed by
+ /// VECREDUCE_FADD on the products. Vectors are operands 0 and 1.
+ SDValue expandVecReduceDot(SDNode *Node, SelectionDAG &DAG) const;
+
/// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
/// Returns true if the expansion was successful.
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index ef5faa17fee27..ec500851058e5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1148,42 +1148,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
}
SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ_FDOT(SDNode *N) {
- // Inline the sequential expansion (VECREDUCE_SEQ_FDOT has acc in operand 0).
- SDLoc dl(N);
- SDValue AccOp = N->getOperand(0);
- SDValue VecAOp = N->getOperand(1);
- SDValue VecBOp = N->getOperand(2);
- SDNodeFlags Flags = N->getFlags();
- EVT VT = VecAOp.getValueType();
- EVT EltVT = VT.getVectorElementType();
- unsigned NumElts = VT.getVectorNumElements();
- SmallVector<SDValue, 8> OpsA, OpsB;
- DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
- DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
- SDValue Res = AccOp;
- if (Flags.hasAllowContract())
- for (unsigned i = 0; i < NumElts; i++)
- Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
- else
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
- Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
- }
- ReplaceValueWith(SDValue(N, 0), Res);
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeqDot(N, DAG));
return SDValue();
}
SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_FDOT(SDNode *N) {
- // Decompose unordered FDOT to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
- SDLoc dl(N);
- SDValue VecAOp = N->getOperand(0);
- SDValue VecBOp = N->getOperand(1);
- SDNodeFlags Flags = N->getFlags();
- EVT VT = VecAOp.getValueType();
- EVT EltVT = VT.getVectorElementType();
- SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecAOp, VecBOp, Flags);
- ReplaceValueWith(SDValue(N, 0),
- DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
return SDValue();
}
@@ -3191,42 +3161,12 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ_FDOT(SDNode *N) {
- // Inline sequential expansion for the SEQ variant (acc in operand 0).
- SDLoc dl(N);
- SDValue AccOp = N->getOperand(0);
- SDValue VecAOp = N->getOperand(1);
- SDValue VecBOp = N->getOperand(2);
- SDNodeFlags Flags = N->getFlags();
- EVT VT = VecAOp.getValueType();
- EVT EltVT = VT.getVectorElementType();
- unsigned NumElts = VT.getVectorNumElements();
- SmallVector<SDValue, 8> OpsA, OpsB;
- DAG.ExtractVectorElements(VecAOp, OpsA, 0, NumElts);
- DAG.ExtractVectorElements(VecBOp, OpsB, 0, NumElts);
- SDValue Res = AccOp;
- if (Flags.hasAllowContract())
- for (unsigned i = 0; i < NumElts; i++)
- Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
- else
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
- Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
- }
- ReplaceValueWith(SDValue(N, 0), Res);
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeqDot(N, DAG));
return SDValue();
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_FDOT(SDNode *N) {
- // Decompose unordered FDOT to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
- SDLoc dl(N);
- SDValue VecAOp = N->getOperand(0);
- SDValue VecBOp = N->getOperand(1);
- SDNodeFlags Flags = N->getFlags();
- EVT VT = VecAOp.getValueType();
- EVT EltVT = VT.getVectorElementType();
- SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecAOp, VecBOp, Flags);
- ReplaceValueWith(SDValue(N, 0),
- DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
+ ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceDot(N, DAG));
return SDValue();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 7d3b6c5d949e2..f91856dfe6a69 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1318,43 +1318,12 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
- case ISD::VECREDUCE_SEQ_FDOT: {
- // Inline sequential expansion (cannot use expandVecReduceSeq — no base opcode).
- SDLoc dl(Node);
- SDValue AccOp = Node->getOperand(0);
- SDValue VecA = Node->getOperand(1), VecB = Node->getOperand(2);
- SDNodeFlags Flags = Node->getFlags();
- EVT VT = VecA.getValueType();
- EVT EltVT = VT.getVectorElementType();
- if (VT.isScalableVector())
- report_fatal_error("Expanding reductions for scalable vectors is undefined.");
- unsigned NumElts = VT.getVectorNumElements();
- SmallVector<SDValue, 8> OpsA, OpsB;
- DAG.ExtractVectorElements(VecA, OpsA, 0, NumElts);
- DAG.ExtractVectorElements(VecB, OpsB, 0, NumElts);
- SDValue Res = AccOp;
- if (Flags.hasAllowContract())
- for (unsigned i = 0; i < NumElts; i++)
- Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
- else
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
- Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
- }
- Results.push_back(Res);
+ case ISD::VECREDUCE_SEQ_FDOT:
+ Results.push_back(TLI.expandVecReduceSeqDot(Node, DAG));
return;
- }
- case ISD::VECREDUCE_FDOT: {
- // Decompose to FMUL(vecA, vecB) + VECREDUCE_FADD(products).
- SDLoc dl(Node);
- SDValue VecA = Node->getOperand(0), VecB = Node->getOperand(1);
- SDNodeFlags Flags = Node->getFlags();
- EVT VT = VecA.getValueType();
- EVT EltVT = VT.getVectorElementType();
- SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecA, VecB, Flags);
- Results.push_back(DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags));
+ case ISD::VECREDUCE_FDOT:
+ Results.push_back(TLI.expandVecReduceDot(Node, DAG));
return;
- }
case ISD::SREM:
case ISD::UREM:
ExpandREM(Node, Results);
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index e6aa222425d13..94485ec2f6dc8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12259,6 +12259,51 @@ SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) cons
return Res;
}
+SDValue TargetLowering::expandVecReduceSeqDot(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ SDValue AccOp = Node->getOperand(0);
+ SDValue VecA = Node->getOperand(1);
+ SDValue VecB = Node->getOperand(2);
+ SDNodeFlags Flags = Node->getFlags();
+
+ EVT VT = VecA.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+
+ if (VT.isScalableVector())
+ report_fatal_error("Expanding reductions for scalable vectors is undefined.");
+
+ unsigned NumElts = VT.getVectorNumElements();
+ SmallVector<SDValue, 8> OpsA, OpsB;
+ DAG.ExtractVectorElements(VecA, OpsA, 0, NumElts);
+ DAG.ExtractVectorElements(VecB, OpsB, 0, NumElts);
+
+ SDValue Res = AccOp;
+ if (Flags.hasAllowContract())
+ for (unsigned i = 0; i < NumElts; i++)
+ Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
+ else
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
+ return Res;
+}
+
+SDValue TargetLowering::expandVecReduceDot(SDNode *Node,
+ SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ SDValue VecA = Node->getOperand(0);
+ SDValue VecB = Node->getOperand(1);
+ SDNodeFlags Flags = Node->getFlags();
+
+ EVT VT = VecA.getValueType();
+ EVT EltVT = VT.getVectorElementType();
+
+ SDValue Products = DAG.getNode(ISD::FMUL, dl, VT, VecA, VecB, Flags);
+ return DAG.getNode(ISD::VECREDUCE_FADD, dl, EltVT, Products, Flags);
+}
+
bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
>From 9ab8a02bca334331150405cbcd28d69270db0841 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:40:16 -0700
Subject: [PATCH 09/17] [GlobalISel] Add legalization strategies for
G_VECREDUCE_{SEQ_}FDOT
Adds fewerElementsVector, moreElementsVector, and widenScalar support
for G_VECREDUCE_FDOT and G_VECREDUCE_SEQ_FDOT in LegalizerHelper.
fewerElementsVectorReductions: splits both vector inputs, computes
partial dot products per chunk, then sums with a tree or sequential
FADD.
fewerElementsVectorSeqReductions: fully scalarizes SEQ_FDOT to a
scalar FMUL+FADD chain, matching the SEQ_FADD/SEQ_FMUL pattern.
moreElementsVector: pads both vector inputs with 0.0 (the neutral
element for dot product) so extra lanes contribute nothing to the sum.
Reuses getNeutralElementForVecReduce.
widenScalar: extends all FP operands and narrows the result via
G_FPEXT/G_FPTRUNC.
Also removes the incorrect lower() handling that was emitting
G_FMUL+G_VECREDUCE_SEQ_FADD; all VECREDUCE opcodes return
UnableToLegalize from lower() and let fewerElements handle it.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 173 +++++++++++++++---
1 file changed, 144 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 3c90775186f47..bb2fa8966905d 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3541,6 +3541,31 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_VECREDUCE_FDOT: {
+ // (dst: sN, vecA: <k x sN>, vecB: <k x sN>) — widen both vectors and dst.
+ if (TypeIdx != 0)
+ return UnableToLegalize;
+ Observer.changingInstr(MI);
+ LLT WideVecTy = MRI.getType(MI.getOperand(1).getReg()).changeElementType(WideTy);
+ widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT);
+ widenScalarSrc(MI, WideVecTy, 2, TargetOpcode::G_FPEXT);
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
+ // (dst: sN, acc: sN, vecA: <k x sN>, vecB: <k x sN>) — widen all FP operands.
+ if (TypeIdx != 0)
+ return UnableToLegalize;
+ Observer.changingInstr(MI);
+ LLT WideVecTy = MRI.getType(MI.getOperand(2).getReg()).changeElementType(WideTy);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); // acc
+ widenScalarSrc(MI, WideVecTy, 2, TargetOpcode::G_FPEXT); // vecA
+ widenScalarSrc(MI, WideVecTy, 3, TargetOpcode::G_FPEXT); // vecB
+ widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
case TargetOpcode::G_VSCALE: {
MachineOperand &SrcMO = MI.getOperand(1);
LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
@@ -4921,21 +4946,6 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
return lowerTRUNC(MI);
GISEL_VECREDUCE_CASES_NONSEQ
return lowerVectorReduction(MI);
- case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
- // G_VECREDUCE_SEQ_FDOT(dst, acc, vecA, vecB) ->
- // G_FMUL(vecA, vecB), G_VECREDUCE_SEQ_FADD(acc, products)
- Register DstReg = MI.getOperand(0).getReg();
- Register AccReg = MI.getOperand(1).getReg();
- Register VecA = MI.getOperand(2).getReg();
- Register VecB = MI.getOperand(3).getReg();
- LLT VecTy = MRI.getType(VecA);
- unsigned Flags = MI.getFlags();
- auto Products = MIRBuilder.buildFMul(VecTy, VecA, VecB, Flags);
- MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FADD, {DstReg},
- {AccReg, Products.getReg(0)}, Flags);
- MI.eraseFromParent();
- return Legalized;
- }
case G_VAARG:
return lowerVAArg(MI);
case G_ATOMICRMW_SUB: {
@@ -5732,6 +5742,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
case TargetOpcode::G_VECREDUCE_SEQ_FADD:
case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT:
return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy);
case G_SHUFFLE_VECTOR:
return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
@@ -5926,6 +5937,64 @@ LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle(
LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
+ if (MI.getOpcode() == TargetOpcode::G_VECREDUCE_FDOT) {
+ if (TypeIdx != 1)
+ return UnableToLegalize;
+
+ Register DstReg = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ Register VecAReg = MI.getOperand(1).getReg();
+ Register VecBReg = MI.getOperand(2).getReg();
+ LLT VecTy = MRI.getType(VecAReg);
+
+ if (NarrowTy.isVector() &&
+ VecTy.getNumElements() % NarrowTy.getNumElements() != 0)
+ return UnableToLegalize;
+
+ const unsigned NumParts =
+ NarrowTy.isVector() ? VecTy.getNumElements() / NarrowTy.getNumElements()
+ : VecTy.getNumElements();
+
+ SmallVector<Register> SplitA, SplitB;
+ extractParts(VecAReg, NarrowTy, NumParts, SplitA, MIRBuilder, MRI);
+ extractParts(VecBReg, NarrowTy, NumParts, SplitB, MIRBuilder, MRI);
+
+ SmallVector<Register> Partials;
+ for (unsigned I = 0; I < NumParts; I++) {
+ if (NarrowTy.isScalar())
+ Partials.push_back(
+ MIRBuilder.buildFMul(DstTy, SplitA[I], SplitB[I], MI.getFlags())
+ .getReg(0));
+ else
+ Partials.push_back(
+ MIRBuilder
+ .buildInstr(TargetOpcode::G_VECREDUCE_FDOT, {DstTy},
+ {SplitA[I], SplitB[I]}, MI.getFlags())
+ .getReg(0));
+ }
+
+ if (isPowerOf2_32(NumParts)) {
+ while (Partials.size() > 1) {
+ SmallVector<Register> Next;
+ for (unsigned I = 0; I + 1 < Partials.size(); I += 2)
+ Next.push_back(
+ MIRBuilder
+ .buildFAdd(DstTy, Partials[I], Partials[I + 1], MI.getFlags())
+ .getReg(0));
+ Partials = Next;
+ }
+ } else {
+ for (unsigned I = 1; I < NumParts; I++)
+ Partials[0] = MIRBuilder
+ .buildFAdd(DstTy, Partials[0], Partials[I],
+ MI.getFlags())
+ .getReg(0);
+ }
+ MIRBuilder.buildCopy(DstReg, Partials[0]);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
auto &RdxMI = cast<GVecReduce>(MI);
if (TypeIdx != 1)
@@ -6014,6 +6083,33 @@ LegalizerHelper::LegalizeResult
LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
unsigned int TypeIdx,
LLT NarrowTy) {
+ if (MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FDOT) {
+ Register DstReg = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ Register AccReg = MI.getOperand(1).getReg();
+ Register VecAReg = MI.getOperand(2).getReg();
+ Register VecBReg = MI.getOperand(3).getReg();
+ LLT VecTy = MRI.getType(VecAReg);
+
+ if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != MRI.getType(AccReg) ||
+ DstTy != NarrowTy)
+ return UnableToLegalize;
+
+ const unsigned NumParts = VecTy.getNumElements();
+ SmallVector<Register> SplitA, SplitB;
+ extractParts(VecAReg, NarrowTy, NumParts, SplitA, MIRBuilder, MRI);
+ extractParts(VecBReg, NarrowTy, NumParts, SplitB, MIRBuilder, MRI);
+
+ Register Acc = AccReg;
+ for (unsigned I = 0; I < NumParts; I++) {
+ auto Mul = MIRBuilder.buildFMul(DstTy, SplitA[I], SplitB[I], MI.getFlags());
+ Acc = MIRBuilder.buildFAdd(DstTy, Acc, Mul, MI.getFlags()).getReg(0);
+ }
+ MIRBuilder.buildCopy(DstReg, Acc);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
MI.getFirst3RegLLTs();
if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
@@ -6689,6 +6785,8 @@ MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
return MIRBuilder.buildFConstant(Ty, -0.0);
case TargetOpcode::G_VECREDUCE_FMUL:
return MIRBuilder.buildFConstant(Ty, 1.0);
+ case TargetOpcode::G_VECREDUCE_FDOT:
+ return MIRBuilder.buildFConstant(Ty, 0.0);
case TargetOpcode::G_VECREDUCE_FMINIMUM:
case TargetOpcode::G_VECREDUCE_FMAXIMUM:
assert(false && "getNeutralElementForVecReduce unimplemented for "
@@ -6927,6 +7025,37 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_VECREDUCE_FDOT:
+ case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
+ bool IsSeq = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FDOT;
+ unsigned VecAIdx = IsSeq ? 2 : 1;
+ unsigned VecBIdx = IsSeq ? 3 : 2;
+ if (TypeIdx != (IsSeq ? 2u : 1u))
+ return UnableToLegalize;
+
+ LLT OrigTy = MRI.getType(MI.getOperand(VecAIdx).getReg());
+ auto NeutralElement = getNeutralElementForVecReduce(
+ TargetOpcode::G_VECREDUCE_FDOT, MIRBuilder, MoreTy.getElementType());
+ LLT IdxTy(TLI.getVectorIdxLLT(MIRBuilder.getDataLayout()));
+
+ auto PadWithZero = [&](unsigned OpIdx) -> Register {
+ MachineOperand &MO = MI.getOperand(OpIdx);
+ auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO);
+ for (size_t I = OrigTy.getNumElements(), E = MoreTy.getNumElements();
+ I != E; I++) {
+ auto Idx = MIRBuilder.buildConstant(IdxTy, I);
+ NewVec = MIRBuilder.buildInsertVectorElement(MoreTy, NewVec,
+ NeutralElement, Idx);
+ }
+ return NewVec.getReg(0);
+ };
+
+ Observer.changingInstr(MI);
+ MI.getOperand(VecAIdx).setReg(PadWithZero(VecAIdx));
+ MI.getOperand(VecBIdx).setReg(PadWithZero(VecBIdx));
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_FMUL:
case TargetOpcode::G_VECREDUCE_ADD:
@@ -10484,20 +10613,6 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) {
LegalizerHelper::LegalizeResult
LegalizerHelper::lowerVectorReduction(MachineInstr &MI) {
- // G_VECREDUCE_FDOT(dst, vecA, vecB) -> G_FMUL(vecA, vecB), G_VECREDUCE_FADD(products)
- if (MI.getOpcode() == TargetOpcode::G_VECREDUCE_FDOT) {
- Register DstReg = MI.getOperand(0).getReg();
- Register VecA = MI.getOperand(1).getReg();
- Register VecB = MI.getOperand(2).getReg();
- LLT VecTy = MRI.getType(VecA);
- unsigned Flags = MI.getFlags();
- auto Products = MIRBuilder.buildFMul(VecTy, VecA, VecB, Flags);
- MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_FADD, {DstReg},
- {Products.getReg(0)}, Flags);
- MI.eraseFromParent();
- return Legalized;
- }
-
Register SrcReg = MI.getOperand(1).getReg();
LLT SrcTy = MRI.getType(SrcReg);
LLT DstTy = MRI.getType(SrcReg);
>From 8c60385b4f317fc0e42841e3d569b7788b16c79e Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:40:23 -0700
Subject: [PATCH 10/17] [SelectionDAG] Rely on DAGCombiner for FMA fusion in
FDOT contract lowering
Instead of explicitly emitting ISD::FMA in expandVecReduceSeqDot and
ScalarizeVecOp_VECREDUCE_SEQ_FDOT when the contract flag is set, always
emit ISD::FMUL + ISD::FADD with the original flags preserved. DAGCombiner
already fuses FMUL+FADD pairs to FMA when the contract flag is present,
regardless of optimization level.
This avoids generating ISD::FMA on targets without native FMA support
(e.g., X86 AVX2 without +fma), which previously caused unnecessary
library calls to fmaf. With the new code, those targets simply emit
vmulss+vaddss.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 4 --
.../CodeGen/SelectionDAG/TargetLowering.cpp | 12 ++---
llvm/test/CodeGen/X86/vector-reduce-fdot.ll | 46 +++++++------------
3 files changed, 20 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 6505627659c2f..9197bb4fb7877 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1248,7 +1248,6 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
}
SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
- // 3-operand sequential: ScalarizeVecOp_VECREDUCE_SEQ_FDOT(acc, a, b) -> FMA or FMUL+FADD.
SDLoc dl(N);
SDValue AccOp = N->getOperand(0);
SDValue VecAOp = N->getOperand(1);
@@ -1259,9 +1258,6 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
SDValue A = GetScalarizedVector(VecAOp);
SDValue B = GetScalarizedVector(VecBOp);
- if (Flags.hasAllowContract())
- return DAG.getNode(ISD::FMA, dl, EltVT, A, B, AccOp, Flags);
-
SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, A, B, Flags);
return DAG.getNode(ISD::FADD, dl, EltVT, AccOp, Mul, Flags);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 94485ec2f6dc8..63c4e531cfb98 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12279,14 +12279,10 @@ SDValue TargetLowering::expandVecReduceSeqDot(SDNode *Node,
DAG.ExtractVectorElements(VecB, OpsB, 0, NumElts);
SDValue Res = AccOp;
- if (Flags.hasAllowContract())
- for (unsigned i = 0; i < NumElts; i++)
- Res = DAG.getNode(ISD::FMA, dl, EltVT, OpsA[i], OpsB[i], Res, Flags);
- else
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
- Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
- }
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, EltVT, OpsA[i], OpsB[i], Flags);
+ Res = DAG.getNode(ISD::FADD, dl, EltVT, Res, Mul, Flags);
+ }
return Res;
}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
index 991a287ac97a6..7b49c3c9914b9 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
@@ -70,39 +70,25 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
-; With contract and FMA hardware: vfmadd chain.
+; With contract: DAGCombiner fuses to vfmadd chain when FMA hardware is available.
define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; AVX2-LABEL: fdot_f32_contract:
; AVX2: # %bb.0:
-; AVX2-NEXT: subq $40, %rsp
-; AVX2-NEXT: .cfi_def_cfa_offset 48
-; AVX2-NEXT: vmovaps %xmm2, %xmm3
-; AVX2-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill
-; AVX2-NEXT: vmovaps %xmm0, %xmm2
-; AVX2-NEXT: vmovaps %xmm1, %xmm0
-; AVX2-NEXT: vmovaps %xmm3, %xmm1
-; AVX2-NEXT: callq fmaf at PLT
-; AVX2-NEXT: vmovaps %xmm0, %xmm2
-; AVX2-NEXT: vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm0 = mem[1,1,3,3]
-; AVX2-NEXT: vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm1 = mem[1,1,3,3]
-; AVX2-NEXT: callq fmaf at PLT
-; AVX2-NEXT: vmovaps %xmm0, %xmm2
-; AVX2-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm0 = mem[1,0]
-; AVX2-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm1 = mem[1,0]
-; AVX2-NEXT: callq fmaf at PLT
-; AVX2-NEXT: vmovapd %xmm0, %xmm2
-; AVX2-NEXT: vpermilps $255, (%rsp), %xmm0 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm0 = mem[3,3,3,3]
-; AVX2-NEXT: vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX2-NEXT: # xmm1 = mem[3,3,3,3]
-; AVX2-NEXT: addq $40, %rsp
-; AVX2-NEXT: .cfi_def_cfa_offset 8
-; AVX2-NEXT: jmp fmaf at PLT # TAILCALL
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX2-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX2-NEXT: vmulss %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX2-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX2-NEXT: vmulss %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
;
; FMA-LABEL: fdot_f32_contract:
; FMA: # %bb.0:
>From 7bca5bacd88cd5bd38da8563d365259817d8c89b Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:43:13 -0700
Subject: [PATCH 11/17] [SelectionDAG] Update VECREDUCE_{SEQ_}FDOT comments in
ISDOpcodes.h
Clarify that both nodes lower to FMUL+FADD sequences; with the
'contract' flag, DAGCombiner may fuse each pair into an FMA.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 33e14e739014a..3445c22bd1087 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1489,9 +1489,8 @@ enum NodeType {
VECREDUCE_SEQ_FMUL,
/// VECREDUCE_SEQ_FDOT(Acc: scalar, VecA, VecB) - Sequential floating-point
/// dot product reduction. Computes Acc + sum(VecA[i] * VecB[i]) in strict
- /// left-to-right order.
- /// Without 'contract' flag: sequential fmul+fadd pairs (two roundings each).
- /// With 'contract' flag: sequential FMA chain (single rounding per element).
+ /// left-to-right order as sequential fmul+fadd pairs. When the 'contract'
+ /// flag is set, DAGCombiner may fuse each fmul+fadd into an FMA.
VECREDUCE_SEQ_FDOT,
/// These reductions have relaxed evaluation order semantics, and have a
@@ -1517,8 +1516,8 @@ enum NodeType {
VECREDUCE_FMINIMUM,
/// VECREDUCE_FDOT(VecA, VecB) - Unordered floating-point dot product
/// reduction. Computes sum(VecA[i] * VecB[i]) with unspecified evaluation
- /// order. The caller adds the accumulator via a separate FADD. Decomposes
- /// to FMUL(VecA, VecB) followed by VECREDUCE_FADD on the products.
+ /// order. Decomposes to a vector FMUL followed by VECREDUCE_FADD on the
+ /// products; the caller adds the accumulator via a separate FADD.
VECREDUCE_FDOT,
/// Integer reductions may have a result type larger than the vector element
/// type. However, the reduction is performed using the vector element type
>From fb5044b1173ddb1466b14846ea13216c872752ca Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:46:30 -0700
Subject: [PATCH 12/17] [ExpandReductions] Drop explicit fma in fdot contract
expansion
Same rationale as the SelectionDAG change: always emit fmul+fadd with
the original fast-math flags preserved. When 'contract' is set, the
backend (DAGCombiner or MachineCombiner) will fuse each pair into an FMA.
The declare for llvm.fma.f32 in the test is no longer needed.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/lib/CodeGen/ExpandReductions.cpp | 11 ++---------
.../CodeGen/Generic/expand-fdot-reduction.ll | 16 ++++++++++------
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index 748fc1229f2ea..8fada07b8587c 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -104,19 +104,12 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
LocalRdx = Builder.CreateBinOp(Instruction::FAdd, Acc, LocalRdx,
"bin.rdx");
} else {
- // Sequential: with 'contract': fma(a[i], b[i], rdx) chain.
- // without: fmul(a[i], b[i]) + fadd(rdx, prod) chain.
LocalRdx = Acc;
for (unsigned i = 0; i < NumElts; i++) {
Value *Ai = Builder.CreateExtractElement(VecA, i);
Value *Bi = Builder.CreateExtractElement(VecB, i);
- if (FMF.allowContract()) {
- LocalRdx = Builder.CreateIntrinsic(
- Intrinsic::fma, {VecTy->getElementType()}, {Ai, Bi, LocalRdx});
- } else {
- Value *Prod = Builder.CreateFMul(Ai, Bi);
- LocalRdx = Builder.CreateFAdd(LocalRdx, Prod);
- }
+ Value *Prod = Builder.CreateFMul(Ai, Bi);
+ LocalRdx = Builder.CreateFAdd(LocalRdx, Prod);
}
}
II->replaceAllUsesWith(LocalRdx);
diff --git a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
index d70b8b116f457..8c2a98dddba7a 100644
--- a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
+++ b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
@@ -2,7 +2,6 @@
; RUN: opt < %s -passes=expand-reductions -S | FileCheck %s
declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
-declare float @llvm.fma.f32(float, float, float)
; Default (no fast-math flags): sequential fmul + fadd chain.
define float @fdot_ordered(float %acc, <4 x float> %a, <4 x float> %b) {
@@ -45,21 +44,26 @@ define float @fdot_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
-; With contract flag: sequential FMA chain.
+; With contract flag: sequential fmul+fadd chain with contract flags preserved;
+; the backend may fuse each pair into an FMA.
define float @fdot_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: @fdot_contract(
; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
-; CHECK-NEXT: [[RDX0:%.*]] = call contract float @llvm.fma.f32(float [[A0]], float [[B0]], float [[ACC:%.*]])
+; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[ACC0:%.*]] = fadd contract float [[ACC:%.*]], [[PROD0]]
; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i64 1
; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i64 1
-; CHECK-NEXT: [[RDX1:%.*]] = call contract float @llvm.fma.f32(float [[A1]], float [[B1]], float [[RDX0]])
+; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[ACC0]], [[PROD1]]
; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i64 2
; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i64 2
-; CHECK-NEXT: [[RDX2:%.*]] = call contract float @llvm.fma.f32(float [[A2]], float [[B2]], float [[RDX1]])
+; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[A2]], [[B2]]
+; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i64 3
-; CHECK-NEXT: [[RDX:%.*]] = call contract float @llvm.fma.f32(float [[A3]], float [[B3]], float [[RDX2]])
+; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[A3]], [[B3]]
+; CHECK-NEXT: [[RDX:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
; CHECK-NEXT: ret float [[RDX]]
;
%res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
>From 464595a19bb0bfedceeb00681130b3b7d3b405d3 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:47:21 -0700
Subject: [PATCH 13/17] [SelectionDAG] Update expandVecReduceSeqDot comment to
drop FMA reference
The function no longer has a special contract->FMA path; update the
docstring to reflect that FMA fusion is left to DAGCombiner.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 55f9e6035018d..465022c465a09 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5863,9 +5863,10 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
- /// Expand a VECREDUCE_SEQ_FDOT into a sequential chain of fmul+fadd (or FMA
- /// with the 'contract' flag). The accumulator is operand 0; vectors are
- /// operands 1 and 2.
+ /// Expand a VECREDUCE_SEQ_FDOT into a sequential fmul+fadd chain with the
+ /// node's fast-math flags preserved. The accumulator is operand 0; vectors
+ /// are operands 1 and 2. DAGCombiner may fuse pairs into FMA if 'contract'
+ /// is set.
SDValue expandVecReduceSeqDot(SDNode *Node, SelectionDAG &DAG) const;
/// Expand a VECREDUCE_FDOT into FMUL(vecA, vecB) followed by
>From 2192d2bd0f28c748bd2522849193724f1b22f391 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Tue, 14 Apr 2026 18:51:44 -0700
Subject: [PATCH 14/17] Update documentation
---
llvm/docs/LangRef.rst | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index aa2cf87ecbf86..43fbcd9b9daad 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -20823,13 +20823,12 @@ attached to the call:
to the running accumulator. Two roundings occur per element (one for the
multiply, one for the add).
-* With the ``contract`` fast-math flag, the computation uses a sequential FMA
- chain: ``fma(%a[0], %b[0], fma(%a[1], %b[1], ... %acc ...))``. Only one
- rounding occurs per element.
+* With the ``contract`` fast-math flag, the computation may use a sequential
+ FMA chain: ``fma(%a[0], %b[0], fma(%a[1], %b[1], ... %acc ...))``.
* With the ``reassoc`` fast-math flag, the order of operations is unspecified.
- Targets may exploit this to perform a tree-based or otherwise reordered
- reduction for improved performance.
+ The computation may become a tree reduction: ``%c = fmul(%a, %b)`` followed
+ by ``@llvm.vector.reduce.fadd(%c)``.
Examples:
"""""""""
>From 219fa4186f196f3f6d38a2dcc670b62238677965 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Wed, 15 Apr 2026 01:56:27 -0700
Subject: [PATCH 15/17] Fix clang-format issues in FDOT SelectionDAG code
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp | 6 ++++--
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 7 ++++---
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 0038241d35b56..91d44c2c78acc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -568,8 +568,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::VECREDUCE_SEQ_FADD: return "vecreduce_seq_fadd";
case ISD::VECREDUCE_FMUL: return "vecreduce_fmul";
case ISD::VECREDUCE_SEQ_FMUL: return "vecreduce_seq_fmul";
- case ISD::VECREDUCE_SEQ_FDOT: return "vecreduce_seq_fdot";
- case ISD::VECREDUCE_FDOT: return "vecreduce_fdot";
+ case ISD::VECREDUCE_SEQ_FDOT:
+ return "vecreduce_seq_fdot";
+ case ISD::VECREDUCE_FDOT:
+ return "vecreduce_fdot";
case ISD::VECREDUCE_ADD: return "vecreduce_add";
case ISD::VECREDUCE_MUL: return "vecreduce_mul";
case ISD::VECREDUCE_AND: return "vecreduce_and";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 63c4e531cfb98..3d3c5a86e5e9b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12260,7 +12260,7 @@ SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) cons
}
SDValue TargetLowering::expandVecReduceSeqDot(SDNode *Node,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG) const {
SDLoc dl(Node);
SDValue AccOp = Node->getOperand(0);
SDValue VecA = Node->getOperand(1);
@@ -12271,7 +12271,8 @@ SDValue TargetLowering::expandVecReduceSeqDot(SDNode *Node,
EVT EltVT = VT.getVectorElementType();
if (VT.isScalableVector())
- report_fatal_error("Expanding reductions for scalable vectors is undefined.");
+ report_fatal_error(
+ "Expanding reductions for scalable vectors is undefined.");
unsigned NumElts = VT.getVectorNumElements();
SmallVector<SDValue, 8> OpsA, OpsB;
@@ -12287,7 +12288,7 @@ SDValue TargetLowering::expandVecReduceSeqDot(SDNode *Node,
}
SDValue TargetLowering::expandVecReduceDot(SDNode *Node,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG) const {
SDLoc dl(Node);
SDValue VecA = Node->getOperand(0);
SDValue VecB = Node->getOperand(1);
>From 1ca20e0fbaffcf74bf212ad9727b2b169c02e421 Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Wed, 15 Apr 2026 02:39:24 -0700
Subject: [PATCH 16/17] Fix clang-format line-length issues in FDOT
implementation
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
.../CodeGen/GlobalISel/MachineIRBuilder.h | 6 ++--
llvm/lib/CodeGen/ExpandReductions.cpp | 8 ++---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 29 ++++++++++---------
llvm/lib/CodeGen/MachineVerifier.cpp | 3 +-
.../SelectionDAG/LegalizeVectorOps.cpp | 3 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 6 ++--
.../SelectionDAG/SelectionDAGBuilder.cpp | 9 +++---
7 files changed, 35 insertions(+), 29 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 6942a7dec2344..30440ea4c0a6c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -2315,8 +2315,7 @@ class LLVM_ABI MachineIRBuilder {
///
/// \p Acc is the scalar accumulator. Computes Acc + sum(VecA[i] * VecB[i])
/// in strict left-to-right order.
- MachineInstrBuilder buildVecReduceSeqFDot(const DstOp &Dst,
- const SrcOp &Acc,
+ MachineInstrBuilder buildVecReduceSeqFDot(const DstOp &Dst, const SrcOp &Acc,
const SrcOp &VecA,
const SrcOp &VecB) {
return buildInstr(TargetOpcode::G_VECREDUCE_SEQ_FDOT, {Dst},
@@ -2326,8 +2325,7 @@ class LLVM_ABI MachineIRBuilder {
/// Build and insert \p Res = G_VECREDUCE_FDOT \p VecA, \p VecB
///
/// Computes sum(VecA[i] * VecB[i]) in unspecified order (no accumulator).
- MachineInstrBuilder buildVecReduceFDot(const DstOp &Dst,
- const SrcOp &VecA,
+ MachineInstrBuilder buildVecReduceFDot(const DstOp &Dst, const SrcOp &VecA,
const SrcOp &VecB) {
return buildInstr(TargetOpcode::G_VECREDUCE_FDOT, {Dst}, {VecA, VecB});
}
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index 8fada07b8587c..0a18bc6140780 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -99,10 +99,10 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
if (FMF.allowReassoc() && isPowerOf2_32(NumElts)) {
// Reassoc + power-of-2: fmul vector, shuffle-tree fadd, then add acc.
Value *Products = Builder.CreateFMul(VecA, VecB);
- LocalRdx = getShuffleReduction(Builder, Products, Instruction::FAdd,
- RS, RK);
- LocalRdx = Builder.CreateBinOp(Instruction::FAdd, Acc, LocalRdx,
- "bin.rdx");
+ LocalRdx =
+ getShuffleReduction(Builder, Products, Instruction::FAdd, RS, RK);
+ LocalRdx =
+ Builder.CreateBinOp(Instruction::FAdd, Acc, LocalRdx, "bin.rdx");
} else {
LocalRdx = Acc;
for (unsigned i = 0; i < NumElts; i++) {
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index bb2fa8966905d..8bab034a7be31 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3546,7 +3546,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
if (TypeIdx != 0)
return UnableToLegalize;
Observer.changingInstr(MI);
- LLT WideVecTy = MRI.getType(MI.getOperand(1).getReg()).changeElementType(WideTy);
+ LLT WideVecTy =
+ MRI.getType(MI.getOperand(1).getReg()).changeElementType(WideTy);
widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT);
widenScalarSrc(MI, WideVecTy, 2, TargetOpcode::G_FPEXT);
widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
@@ -3554,11 +3555,13 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return Legalized;
}
case TargetOpcode::G_VECREDUCE_SEQ_FDOT: {
- // (dst: sN, acc: sN, vecA: <k x sN>, vecB: <k x sN>) — widen all FP operands.
+ // (dst: sN, acc: sN, vecA: <k x sN>, vecB: <k x sN>) — widen all FP
+ // operands.
if (TypeIdx != 0)
return UnableToLegalize;
Observer.changingInstr(MI);
- LLT WideVecTy = MRI.getType(MI.getOperand(2).getReg()).changeElementType(WideTy);
+ LLT WideVecTy =
+ MRI.getType(MI.getOperand(2).getReg()).changeElementType(WideTy);
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); // acc
widenScalarSrc(MI, WideVecTy, 2, TargetOpcode::G_FPEXT); // vecA
widenScalarSrc(MI, WideVecTy, 3, TargetOpcode::G_FPEXT); // vecB
@@ -5966,11 +5969,11 @@ LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
MIRBuilder.buildFMul(DstTy, SplitA[I], SplitB[I], MI.getFlags())
.getReg(0));
else
- Partials.push_back(
- MIRBuilder
- .buildInstr(TargetOpcode::G_VECREDUCE_FDOT, {DstTy},
- {SplitA[I], SplitB[I]}, MI.getFlags())
- .getReg(0));
+ Partials.push_back(MIRBuilder
+ .buildInstr(TargetOpcode::G_VECREDUCE_FDOT,
+ {DstTy}, {SplitA[I], SplitB[I]},
+ MI.getFlags())
+ .getReg(0));
}
if (isPowerOf2_32(NumParts)) {
@@ -5985,10 +5988,9 @@ LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions(
}
} else {
for (unsigned I = 1; I < NumParts; I++)
- Partials[0] = MIRBuilder
- .buildFAdd(DstTy, Partials[0], Partials[I],
- MI.getFlags())
- .getReg(0);
+ Partials[0] =
+ MIRBuilder.buildFAdd(DstTy, Partials[0], Partials[I], MI.getFlags())
+ .getReg(0);
}
MIRBuilder.buildCopy(DstReg, Partials[0]);
MI.eraseFromParent();
@@ -6102,7 +6104,8 @@ LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI,
Register Acc = AccReg;
for (unsigned I = 0; I < NumParts; I++) {
- auto Mul = MIRBuilder.buildFMul(DstTy, SplitA[I], SplitB[I], MI.getFlags());
+ auto Mul =
+ MIRBuilder.buildFMul(DstTy, SplitA[I], SplitB[I], MI.getFlags());
Acc = MIRBuilder.buildFAdd(DstTy, Acc, Mul, MI.getFlags()).getReg(0);
}
MIRBuilder.buildCopy(DstReg, Acc);
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 7cd5b26c1e7fb..38fbc0c4dd1f8 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -2166,7 +2166,8 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
if (!VecBTy.isVector())
report("G_VECREDUCE_SEQ_FDOT requires a vector 3rd operand", MI);
if (VecATy != VecBTy)
- report("G_VECREDUCE_SEQ_FDOT vector operands must have the same type", MI);
+ report("G_VECREDUCE_SEQ_FDOT vector operands must have the same type",
+ MI);
break;
}
case TargetOpcode::G_VECREDUCE_FDOT: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index f91856dfe6a69..2f032efeb9bec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -532,7 +532,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
- case ISD::VECREDUCE_SEQ_FDOT: // 3-operand sequential: action from getOperand(1)
+ case ISD::VECREDUCE_SEQ_FDOT: // 3-operand sequential: action from
+ // getOperand(1)
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 9197bb4fb7877..beca463b633ee 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1263,7 +1263,8 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
}
SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_FDOT(SDNode *N) {
- // 2-operand unordered: ScalarizeVecOp_VECREDUCE_FDOT(a, b) -> FMUL(a[0], b[0]).
+ // 2-operand unordered: ScalarizeVecOp_VECREDUCE_FDOT(a, b) -> FMUL(a[0],
+ // b[0]).
SDLoc dl(N);
SDValue VecAOp = N->getOperand(0);
SDValue VecBOp = N->getOperand(1);
@@ -4050,7 +4051,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ_FDOT(SDNode *N) {
SDValue Partial =
DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, ResVT, AccOp, LoA, LoB, Flags);
// Reduce the upper half with the lower-half result as the new accumulator.
- return DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, ResVT, Partial, HiA, HiB, Flags);
+ return DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, ResVT, Partial, HiA, HiB,
+ Flags);
}
SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_FDOT(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 50080e7bb9292..0dfeb32f4ccb6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -11212,12 +11212,13 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
case Intrinsic::vector_reduce_fdot:
if (SDFlags.hasAllowReassociation())
// Non-sequential: acc added externally, dot product is unordered.
- Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
- DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, Op2, Op3, SDFlags),
- SDFlags);
+ Res = DAG.getNode(
+ ISD::FADD, dl, VT, Op1,
+ DAG.getNode(ISD::VECREDUCE_FDOT, dl, VT, Op2, Op3, SDFlags), SDFlags);
else
// Sequential: VECREDUCE_SEQ_FDOT(acc, vecA, vecB).
- Res = DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, VT, {Op1, Op2, Op3}, SDFlags);
+ Res = DAG.getNode(ISD::VECREDUCE_SEQ_FDOT, dl, VT, {Op1, Op2, Op3},
+ SDFlags);
break;
default:
llvm_unreachable("Unhandled vector reduce intrinsic");
>From ed6b11a07dbe9f0c9407aae33acfc226da87343a Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Thu, 16 Apr 2026 03:54:56 -0700
Subject: [PATCH 17/17] [Tests] Expand fdot test coverage: f16 variants and
reassoc+contract
Add for each fdot test file:
- f16 contract, reassoc, and reassoc+contract variants (AArch64, NVPTX,
Thumb2/MVE, Generic/ExpandReductions)
- f32 reassoc+contract combined variant (all files)
Key observations captured by the tests:
- `reassoc` takes priority over `contract` when both are set; uses the
unordered tree-reduction path (vector fmul + shuffle tree fadd).
- On targets with FMA (AMDGPU, NVPTX, AArch64 O0), the `contract` flag
causes DAGCombiner to fuse the leaf fmul+fadd pairs in the tree into FMAs.
- On X86 with AVX2+FMA, the vector tree reduction prevents scalar FMA fusion;
vmulps+vaddps tree is emitted regardless of the `contract` flag.
- f16 on AMDGPU/NVPTX uses native half-precision instructions; f16 on
AArch64 without fullfp16 soft-promotes through single-precision.
CHECK lines generated with update_llc_test_checks.py / update_test_checks.py
using a build of this branch.
Co-Authored-By: Claude Sonnet 4.6 <noreply at anthropic.com>
---
llvm/test/CodeGen/AArch64/vecreduce-fdot.ll | 240 ++++++++++++++++++
.../test/CodeGen/AMDGPU/vector-reduce-fdot.ll | 15 ++
.../CodeGen/Generic/expand-fdot-reduction.ll | 165 +++++++++---
.../CodeGen/NVPTX/reduction-intrinsics.ll | 115 +++++++++
.../test/CodeGen/Thumb2/mve-vecreduce-fdot.ll | 111 ++++++++
llvm/test/CodeGen/X86/vector-reduce-fdot.ll | 26 ++
6 files changed, 638 insertions(+), 34 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
index 4b9410ca0e627..3df6f0fdd6b4e 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fdot.ll
@@ -101,6 +101,28 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
+define float @fdot_f32_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; O0-LABEL: fdot_f32_reassoc_contract:
+; O0: // %bb.0:
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: faddp v1.4s, v1.4s, v1.4s
+; O0-NEXT: fmov x0, d1
+; O0-NEXT: fmov d1, x0
+; O0-NEXT: faddp s1, v1.2s
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f32_reassoc_contract:
+; O1: // %bb.0:
+; O1-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O1-NEXT: faddp v1.4s, v1.4s, v1.4s
+; O1-NEXT: faddp s1, v1.2s
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: ret
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
; O0-LABEL: fdot_f16:
; O0: // %bb.0:
@@ -196,3 +218,221 @@ define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
%res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
ret half %res
}
+
+define half @fdot_f16_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; O0-LABEL: fdot_f16_contract:
+; O0: // %bb.0:
+; O0-NEXT: // implicit-def: $q5
+; O0-NEXT: fmov d5, d1
+; O0-NEXT: mov h1, v5.h[3]
+; O0-NEXT: // implicit-def: $q16
+; O0-NEXT: fmov d16, d2
+; O0-NEXT: mov h2, v16.h[3]
+; O0-NEXT: mov h3, v5.h[2]
+; O0-NEXT: mov h4, v16.h[2]
+; O0-NEXT: fmov s6, s5
+; O0-NEXT: fmov s7, s16
+; O0-NEXT: mov h5, v5.h[1]
+; O0-NEXT: mov h16, v16.h[1]
+; O0-NEXT: fcvt s16, h16
+; O0-NEXT: fcvt s5, h5
+; O0-NEXT: fmul s5, s5, s16
+; O0-NEXT: fcvt h5, s5
+; O0-NEXT: fcvt s5, h5
+; O0-NEXT: fcvt s7, h7
+; O0-NEXT: fcvt s6, h6
+; O0-NEXT: fmul s6, s6, s7
+; O0-NEXT: fcvt h6, s6
+; O0-NEXT: fcvt s6, h6
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fadd s0, s0, s6
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fadd s0, s0, s5
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s4, h4
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fmul s3, s3, s4
+; O0-NEXT: fcvt h3, s3
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fadd s0, s0, s3
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fcvt s2, h2
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fmul s1, s1, s2
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f16_contract:
+; O1: // %bb.0:
+; O1-NEXT: // kill: def $d2 killed $d2 def $q2
+; O1-NEXT: // kill: def $d1 killed $d1 def $q1
+; O1-NEXT: fcvt s3, h2
+; O1-NEXT: fcvt s4, h1
+; O1-NEXT: mov h5, v2.h[1]
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul s3, s4, s3
+; O1-NEXT: mov h4, v1.h[1]
+; O1-NEXT: fcvt s5, h5
+; O1-NEXT: fcvt h3, s3
+; O1-NEXT: fcvt s4, h4
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fmul s4, s4, s5
+; O1-NEXT: mov h5, v2.h[2]
+; O1-NEXT: mov h2, v2.h[3]
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fcvt h3, s4
+; O1-NEXT: mov h4, v1.h[2]
+; O1-NEXT: fcvt s5, h5
+; O1-NEXT: mov h1, v1.h[3]
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fcvt s4, h4
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul s1, s1, s2
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fmul s3, s4, s5
+; O1-NEXT: fcvt h1, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt h3, s3
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fadd s0, s0, s3
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: ret
+ %res = call contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define half @fdot_f16_reassoc(half %acc, <4 x half> %a, <4 x half> %b) {
+; O0-LABEL: fdot_f16_reassoc:
+; O0: // %bb.0:
+; O0-NEXT: fcvtl v2.4s, v2.4h
+; O0-NEXT: fcvtl v1.4s, v1.4h
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: fcvtn v1.4h, v1.4s
+; O0-NEXT: // implicit-def: $q2
+; O0-NEXT: fmov d2, d1
+; O0-NEXT: mov h1, v2.h[1]
+; O0-NEXT: fcvt s3, h1
+; O0-NEXT: fmov s1, s2
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fadd s1, s1, s3
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: mov h3, v2.h[2]
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fadd s1, s1, s3
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: mov h2, v2.h[3]
+; O0-NEXT: fcvt s2, h2
+; O0-NEXT: fadd s1, s1, s2
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f16_reassoc:
+; O1: // %bb.0:
+; O1-NEXT: fcvtl v2.4s, v2.4h
+; O1-NEXT: fcvtl v1.4s, v1.4h
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O1-NEXT: fcvtn v1.4h, v1.4s
+; O1-NEXT: mov h2, v1.h[1]
+; O1-NEXT: fcvt s3, h1
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s2, s3, s2
+; O1-NEXT: mov h3, v1.h[2]
+; O1-NEXT: mov h1, v1.h[3]
+; O1-NEXT: fcvt h2, s2
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s2, s2, s3
+; O1-NEXT: fcvt h2, s2
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s1, s2, s1
+; O1-NEXT: fcvt h1, s1
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: ret
+ %res = call reassoc half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define half @fdot_f16_reassoc_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; O0-LABEL: fdot_f16_reassoc_contract:
+; O0: // %bb.0:
+; O0-NEXT: fcvtl v2.4s, v2.4h
+; O0-NEXT: fcvtl v1.4s, v1.4h
+; O0-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O0-NEXT: fcvtn v1.4h, v1.4s
+; O0-NEXT: // implicit-def: $q2
+; O0-NEXT: fmov d2, d1
+; O0-NEXT: mov h1, v2.h[1]
+; O0-NEXT: fcvt s3, h1
+; O0-NEXT: fmov s1, s2
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fadd s1, s1, s3
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: mov h3, v2.h[2]
+; O0-NEXT: fcvt s3, h3
+; O0-NEXT: fadd s1, s1, s3
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: mov h2, v2.h[3]
+; O0-NEXT: fcvt s2, h2
+; O0-NEXT: fadd s1, s1, s2
+; O0-NEXT: fcvt h1, s1
+; O0-NEXT: fcvt s1, h1
+; O0-NEXT: fcvt s0, h0
+; O0-NEXT: fadd s0, s0, s1
+; O0-NEXT: fcvt h0, s0
+; O0-NEXT: ret
+;
+; O1-LABEL: fdot_f16_reassoc_contract:
+; O1: // %bb.0:
+; O1-NEXT: fcvtl v2.4s, v2.4h
+; O1-NEXT: fcvtl v1.4s, v1.4h
+; O1-NEXT: fcvt s0, h0
+; O1-NEXT: fmul v1.4s, v1.4s, v2.4s
+; O1-NEXT: fcvtn v1.4h, v1.4s
+; O1-NEXT: mov h2, v1.h[1]
+; O1-NEXT: fcvt s3, h1
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s2, s3, s2
+; O1-NEXT: mov h3, v1.h[2]
+; O1-NEXT: mov h1, v1.h[3]
+; O1-NEXT: fcvt h2, s2
+; O1-NEXT: fcvt s3, h3
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s2, s2, s3
+; O1-NEXT: fcvt h2, s2
+; O1-NEXT: fcvt s2, h2
+; O1-NEXT: fadd s1, s2, s1
+; O1-NEXT: fcvt h1, s1
+; O1-NEXT: fcvt s1, h1
+; O1-NEXT: fadd s0, s0, s1
+; O1-NEXT: fcvt h0, s0
+; O1-NEXT: ret
+ %res = call reassoc contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
index 11ef7e81ccfe8..b0c1f90dcc951 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fdot.ll
@@ -52,3 +52,18 @@ define float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
%res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+; With reassoc+contract: reassoc takes priority (tree reduction with v_mul_f32 +
+; v_add_f32); contract flag preserved but doesn't change tree structure.
+define float @fdot_f32_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; GFX9-LABEL: fdot_f32_reassoc_contract:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_fma_f32 v0, v4, v8, v0
+; GFX9-NEXT: v_fma_f32 v0, v2, v6, v0
+; GFX9-NEXT: v_fma_f32 v0, v3, v7, v0
+; GFX9-NEXT: v_fma_f32 v0, v1, v5, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
diff --git a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
index 8c2a98dddba7a..57f632d1d11ee 100644
--- a/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
+++ b/llvm/test/CodeGen/Generic/expand-fdot-reduction.ll
@@ -2,27 +2,28 @@
; RUN: opt < %s -passes=expand-reductions -S | FileCheck %s
declare float @llvm.vector.reduce.fdot.v4f32(float, <4 x float>, <4 x float>)
+declare half @llvm.vector.reduce.fdot.v4f16(half, <4 x half>, <4 x half>)
; Default (no fast-math flags): sequential fmul + fadd chain.
define float @fdot_ordered(float %acc, <4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: @fdot_ordered(
-; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
-; CHECK-NEXT: [[PROD0:%.*]] = fmul float [[A0]], [[B0]]
-; CHECK-NEXT: [[ACC0:%.*]] = fadd float [[ACC:%.*]], [[PROD0]]
-; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i64 1
-; CHECK-NEXT: [[PROD1:%.*]] = fmul float [[A1]], [[B1]]
-; CHECK-NEXT: [[ACC1:%.*]] = fadd float [[ACC0]], [[PROD1]]
-; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i64 2
-; CHECK-NEXT: [[PROD2:%.*]] = fmul float [[A2]], [[B2]]
-; CHECK-NEXT: [[ACC2:%.*]] = fadd float [[ACC1]], [[PROD2]]
-; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i64 3
-; CHECK-NEXT: [[PROD3:%.*]] = fmul float [[A3]], [[B3]]
-; CHECK-NEXT: [[RDX:%.*]] = fadd float [[ACC2]], [[PROD3]]
-; CHECK-NEXT: ret float [[RDX]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd float [[ACC:%.*]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[B]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd float [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[B]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = fmul float [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fadd float [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x float> [[B]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = fmul float [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd float [[TMP12]], [[TMP15]]
+; CHECK-NEXT: ret float [[TMP16]]
;
%res = call float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
@@ -44,28 +45,124 @@ define float @fdot_reassoc(float %acc, <4 x float> %a, <4 x float> %b) {
ret float %res
}
+; With reassoc+contract: reassoc takes priority, so vector fmul then shuffle-tree
+; fadd; both flags preserved on the generated operations.
+define float @fdot_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @fdot_reassoc_contract(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc contract <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <4 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX2:%.*]] = fadd reassoc contract <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = fadd reassoc contract float [[ACC:%.*]], [[TMP2]]
+; CHECK-NEXT: ret float [[BIN_RDX3]]
+;
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
; With contract flag: sequential fmul+fadd chain with contract flags preserved;
; the backend may fuse each pair into an FMA.
define float @fdot_contract(float %acc, <4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: @fdot_contract(
-; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
-; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[A0]], [[B0]]
-; CHECK-NEXT: [[ACC0:%.*]] = fadd contract float [[ACC:%.*]], [[PROD0]]
-; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i64 1
-; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[A1]], [[B1]]
-; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[ACC0]], [[PROD1]]
-; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i64 2
-; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i64 2
-; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[A2]], [[B2]]
-; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]]
-; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
-; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i64 3
-; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[A3]], [[B3]]
-; CHECK-NEXT: [[RDX:%.*]] = fadd contract float [[ACC2]], [[PROD3]]
-; CHECK-NEXT: ret float [[RDX]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract float [[ACC:%.*]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[A]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[B]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = fmul contract float [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd contract float [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x float> [[A]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[B]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = fmul contract float [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fadd contract float [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[A]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x float> [[B]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract float [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd contract float [[TMP12]], [[TMP15]]
+; CHECK-NEXT: ret float [[TMP16]]
;
%res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+; f16 variants: same expansion patterns as f32, but for half elements.
+define half @fdot_f16_ordered(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: @fdot_f16_ordered(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x half> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul half [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd half [[ACC:%.*]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x half> [[A]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x half> [[B]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = fmul half [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd half [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x half> [[A]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x half> [[B]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = fmul half [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fadd half [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x half> [[A]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x half> [[B]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = fmul half [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd half [[TMP12]], [[TMP15]]
+; CHECK-NEXT: ret half [[TMP16]]
+;
+ %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define half @fdot_f16_reassoc(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: @fdot_f16_reassoc(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc <4 x half> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x half> [[TMP1]], <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc <4 x half> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x half> [[BIN_RDX]], <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX2:%.*]] = fadd reassoc <4 x half> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x half> [[BIN_RDX2]], i32 0
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = fadd reassoc half [[ACC:%.*]], [[TMP2]]
+; CHECK-NEXT: ret half [[BIN_RDX3]]
+;
+ %res = call reassoc half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define half @fdot_f16_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: @fdot_f16_contract(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x half> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract half [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract half [[ACC:%.*]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x half> [[A]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x half> [[B]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = fmul contract half [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd contract half [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x half> [[A]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x half> [[B]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = fmul contract half [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fadd contract half [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x half> [[A]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x half> [[B]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract half [[TMP13]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd contract half [[TMP12]], [[TMP15]]
+; CHECK-NEXT: ret half [[TMP16]]
+;
+ %res = call contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define half @fdot_f16_reassoc_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: @fdot_f16_reassoc_contract(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc contract <4 x half> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x half> [[TMP1]], <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <4 x half> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <4 x half> [[BIN_RDX]], <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[BIN_RDX2:%.*]] = fadd reassoc contract <4 x half> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x half> [[BIN_RDX2]], i32 0
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = fadd reassoc contract half [[ACC:%.*]], [[TMP2]]
+; CHECK-NEXT: ret half [[BIN_RDX3]]
+;
+ %res = call reassoc contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
index 437bf500a5de2..d37ca4a291fa4 100644
--- a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -2732,3 +2732,118 @@ define half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
%res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
ret half %res
}
+
+; f32 reassoc+contract: reassoc takes priority (tree reduction).
+define float @fdot_f32_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-SM80-LABEL: fdot_f32_reassoc_contract(
+; CHECK-SM80: {
+; CHECK-SM80-NEXT: .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT: // %bb.0:
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r6, %r7, %r8, %r9}, [fdot_f32_reassoc_contract_param_2];
+; CHECK-SM80-NEXT: ld.param.v4.b32 {%r2, %r3, %r4, %r5}, [fdot_f32_reassoc_contract_param_1];
+; CHECK-SM80-NEXT: ld.param.b32 %r1, [fdot_f32_reassoc_contract_param_0];
+; CHECK-SM80-NEXT: fma.rn.f32 %r10, %r5, %r9, %r1;
+; CHECK-SM80-NEXT: fma.rn.f32 %r11, %r4, %r8, %r10;
+; CHECK-SM80-NEXT: fma.rn.f32 %r12, %r3, %r7, %r11;
+; CHECK-SM80-NEXT: fma.rn.f32 %r13, %r2, %r6, %r12;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT: ret;
+;
+; CHECK-SM100-LABEL: fdot_f32_reassoc_contract(
+; CHECK-SM100: {
+; CHECK-SM100-NEXT: .reg .b32 %r<10>;
+; CHECK-SM100-NEXT: .reg .b64 %rd<7>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT: // %bb.0:
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fdot_f32_reassoc_contract_param_2];
+; CHECK-SM100-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fdot_f32_reassoc_contract_param_1];
+; CHECK-SM100-NEXT: ld.param.b32 %r1, [fdot_f32_reassoc_contract_param_0];
+; CHECK-SM100-NEXT: mul.f32x2 %rd5, %rd2, %rd4;
+; CHECK-SM100-NEXT: mov.b64 {%r2, %r3}, %rd5;
+; CHECK-SM100-NEXT: add.f32 %r4, %r2, %r3;
+; CHECK-SM100-NEXT: mul.f32x2 %rd6, %rd1, %rd3;
+; CHECK-SM100-NEXT: mov.b64 {%r5, %r6}, %rd6;
+; CHECK-SM100-NEXT: add.f32 %r7, %r5, %r6;
+; CHECK-SM100-NEXT: add.f32 %r8, %r7, %r4;
+; CHECK-SM100-NEXT: add.f32 %r9, %r1, %r8;
+; CHECK-SM100-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM100-NEXT: ret;
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+; f16 contract: sequential fma chain.
+define half @fdot_f16_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_contract(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<14>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_contract_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_contract_param_1];
+; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_contract_param_0];
+; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r1;
+; CHECK-NEXT: mov.b32 {%rs4, %rs5}, %r3;
+; CHECK-NEXT: fma.rn.f16 %rs6, %rs2, %rs4, %rs1;
+; CHECK-NEXT: fma.rn.f16 %rs7, %rs3, %rs5, %rs6;
+; CHECK-NEXT: mov.b32 {%rs8, %rs9}, %r2;
+; CHECK-NEXT: mov.b32 {%rs10, %rs11}, %r4;
+; CHECK-NEXT: fma.rn.f16 %rs12, %rs8, %rs10, %rs7;
+; CHECK-NEXT: fma.rn.f16 %rs13, %rs9, %rs11, %rs12;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs13;
+; CHECK-NEXT: ret;
+ %res = call contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+; f16 reassoc: tree-reduction mul + add chain (unordered).
+define half @fdot_f16_reassoc(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_reassoc(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<10>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_reassoc_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_reassoc_param_1];
+; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_reassoc_param_0];
+; CHECK-NEXT: mul.rn.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r5;
+; CHECK-NEXT: add.rn.f16 %rs4, %rs2, %rs3;
+; CHECK-NEXT: mul.rn.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r6;
+; CHECK-NEXT: add.rn.f16 %rs7, %rs5, %rs6;
+; CHECK-NEXT: add.rn.f16 %rs8, %rs7, %rs4;
+; CHECK-NEXT: add.rn.f16 %rs9, %rs1, %rs8;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-NEXT: ret;
+ %res = call reassoc half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+; f16 reassoc+contract: reassoc takes priority (tree reduction).
+define half @fdot_f16_reassoc_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_reassoc_contract(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<10>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [fdot_f16_reassoc_contract_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [fdot_f16_reassoc_contract_param_1];
+; CHECK-NEXT: ld.param.b16 %rs1, [fdot_f16_reassoc_contract_param_0];
+; CHECK-NEXT: mul.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT: mov.b32 {%rs2, %rs3}, %r5;
+; CHECK-NEXT: add.f16 %rs4, %rs2, %rs3;
+; CHECK-NEXT: mul.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r6;
+; CHECK-NEXT: add.f16 %rs7, %rs5, %rs6;
+; CHECK-NEXT: add.f16 %rs8, %rs7, %rs4;
+; CHECK-NEXT: add.f16 %rs9, %rs1, %rs8;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-NEXT: ret;
+ %res = call reassoc contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
index f49efb1cb18ff..7bc5e6b731065 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-fdot.ll
@@ -43,3 +43,114 @@ define arm_aapcs_vfpcc float @fdot_f32_reassoc(float %acc, <4 x float> %a, <4 x
%res = call reassoc float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+define arm_aapcs_vfpcc float @fdot_f32_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fdot_f32_reassoc_contract:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vmul.f32 q1, q1, q2
+; CHECK-NEXT: vadd.f32 s2, s6, s7
+; CHECK-NEXT: vadd.f32 s4, s4, s5
+; CHECK-NEXT: vadd.f32 s2, s4, s2
+; CHECK-NEXT: vadd.f32 s0, s0, s2
+; CHECK-NEXT: bx lr
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
+
+define arm_aapcs_vfpcc half @fdot_f16(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vmla.f16 s0, s4, s8
+; CHECK-NEXT: vmovx.f16 s2, s8
+; CHECK-NEXT: vmovx.f16 s4, s4
+; CHECK-NEXT: vmla.f16 s0, s4, s2
+; CHECK-NEXT: vmovx.f16 s2, s9
+; CHECK-NEXT: vmla.f16 s0, s5, s9
+; CHECK-NEXT: vmovx.f16 s4, s5
+; CHECK-NEXT: vmla.f16 s0, s4, s2
+; CHECK-NEXT: vldr.16 s2, .LCPI4_0
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: bx lr
+; CHECK-NEXT: .p2align 1
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI4_0:
+; CHECK-NEXT: .short 0x0000 @ half 0
+ %res = call half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define arm_aapcs_vfpcc half @fdot_f16_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_contract:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vfma.f16 s0, s4, s8
+; CHECK-NEXT: vmovx.f16 s2, s8
+; CHECK-NEXT: vmovx.f16 s4, s4
+; CHECK-NEXT: vfma.f16 s0, s4, s2
+; CHECK-NEXT: vmovx.f16 s2, s9
+; CHECK-NEXT: vfma.f16 s0, s5, s9
+; CHECK-NEXT: vmovx.f16 s4, s5
+; CHECK-NEXT: vfma.f16 s0, s4, s2
+; CHECK-NEXT: vldr.16 s2, .LCPI5_0
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: bx lr
+; CHECK-NEXT: .p2align 1
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI5_0:
+; CHECK-NEXT: .short 0x0000 @ half 0
+ %res = call contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define arm_aapcs_vfpcc half @fdot_f16_reassoc(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_reassoc:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vldr.16 s10, .LCPI6_0
+; CHECK-NEXT: vins.f16 s10, s10
+; CHECK-NEXT: vmov.f32 s11, s10
+; CHECK-NEXT: vmov.f32 s6, s10
+; CHECK-NEXT: vmov.f32 s7, s10
+; CHECK-NEXT: vmul.f16 q1, q1, q2
+; CHECK-NEXT: vrev32.16 q2, q1
+; CHECK-NEXT: vadd.f16 q1, q1, q2
+; CHECK-NEXT: vadd.f16 s2, s6, s7
+; CHECK-NEXT: vadd.f16 s4, s4, s5
+; CHECK-NEXT: vadd.f16 s2, s4, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: bx lr
+; CHECK-NEXT: .p2align 1
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI6_0:
+; CHECK-NEXT: .short 0x0000 @ half 0
+ %res = call reassoc half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
+
+define arm_aapcs_vfpcc half @fdot_f16_reassoc_contract(half %acc, <4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: fdot_f16_reassoc_contract:
+; CHECK: @ %bb.0:
+; CHECK-NEXT: vldr.16 s10, .LCPI7_0
+; CHECK-NEXT: vins.f16 s10, s10
+; CHECK-NEXT: vmov.f32 s11, s10
+; CHECK-NEXT: vmov.f32 s6, s10
+; CHECK-NEXT: vmov.f32 s7, s10
+; CHECK-NEXT: vmul.f16 q1, q1, q2
+; CHECK-NEXT: vrev32.16 q2, q1
+; CHECK-NEXT: vadd.f16 q1, q1, q2
+; CHECK-NEXT: vadd.f16 s2, s6, s7
+; CHECK-NEXT: vadd.f16 s4, s4, s5
+; CHECK-NEXT: vadd.f16 s2, s4, s2
+; CHECK-NEXT: vadd.f16 s0, s0, s2
+; CHECK-NEXT: bx lr
+; CHECK-NEXT: .p2align 1
+; CHECK-NEXT: @ %bb.1:
+; CHECK-NEXT: .LCPI7_0:
+; CHECK-NEXT: .short 0x0000 @ half 0
+ %res = call reassoc contract half @llvm.vector.reduce.fdot.v4f16(half %acc, <4 x half> %a, <4 x half> %b)
+ ret half %res
+}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
index 7b49c3c9914b9..d4f42c9676b1b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fdot.ll
@@ -106,3 +106,29 @@ define float @fdot_f32_contract(float %acc, <4 x float> %a, <4 x float> %b) {
%res = call contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
ret float %res
}
+
+; With reassoc+contract: reassoc takes priority (tree reduction); contract flag
+; preserved, backend may fuse muls+adds into FMAs within the tree.
+define float @fdot_f32_reassoc_contract(float %acc, <4 x float> %a, <4 x float> %b) {
+; AVX2-LABEL: fdot_f32_reassoc_contract:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmulps %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX2-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; FMA-LABEL: fdot_f32_reassoc_contract:
+; FMA: # %bb.0:
+; FMA-NEXT: vmulps %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; FMA-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; FMA-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; FMA-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; FMA-NEXT: retq
+ %res = call reassoc contract float @llvm.vector.reduce.fdot.v4f32(float %acc, <4 x float> %a, <4 x float> %b)
+ ret float %res
+}
More information about the llvm-commits
mailing list