[llvm] [MachineCombiner]Reassociate FP accumulation chains to expose ILP (PR #222676)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 07:47:36 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/222676
Add a target-independent machine combiner pattern, FMA_CHAIN, that
rewrites a chain of FP multiply-accumulations (fused FMA, or unfused
FADD over FMUL) ending at a root into two shorter sub-chains combined
by a single add, roughly halving the chain's critical-path latency. The
split requires the reassoc and nsz flags and no FP exceptions, and is
applied only when it reduces the critical path.
Targets opt in via the new TargetInstrInfo::getFMAChainLinkInfo hook,
which reports the plain-multiply and add opcodes and the accumulator
operand index of a chain link. Implemented for X86 (VFMADD, all widths,
register and memory-folded forms), AArch64 (FMADD, FMLA, and unfused
FADD), and RISC-V (scalar FMADD).
Based on PPC support, existing in LLVM, but extending it for other
targets.
Assisted-by: Cursor
>From acc42b37abb03e158311739e986fddb4844989e9 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 10 Sep 2026 07:47:23 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/CodeGen/MachineCombinerPattern.h | 3 +
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 42 +++
llvm/lib/CodeGen/MachineCombiner.cpp | 12 +-
llvm/lib/CodeGen/TargetInstrInfo.cpp | 271 ++++++++++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 45 +++
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 3 +
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 24 ++
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 3 +
llvm/lib/Target/X86/X86InstrInfo.cpp | 80 ++++
llvm/lib/Target/X86/X86InstrInfo.h | 3 +
.../AArch64/machine-combiner-fma-chain.ll | 353 ++++++++++++++++++
.../RISCV/machine-combiner-fma-chain.ll | 113 ++++++
.../CodeGen/X86/machine-combiner-fma-chain.ll | 329 ++++++++++++++++
13 files changed, 1279 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/machine-combiner-fma-chain.ll
create mode 100644 llvm/test/CodeGen/RISCV/machine-combiner-fma-chain.ll
create mode 100644 llvm/test/CodeGen/X86/machine-combiner-fma-chain.ll
diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
index 25fce679323ee..b37ae84f8f77f 100644
--- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
+++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
@@ -33,6 +33,9 @@ enum MachineCombinerPattern : unsigned {
REASSOC_XA_BY,
REASSOC_XA_YB,
ACC_CHAIN,
+ // Reassociation of an FP accumulation (FMA) chain into two shorter chains
+ // combined by a single add, to expose more ILP.
+ FMA_CHAIN,
TARGET_PATTERN_START
};
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index f27261e7be1aa..2e662c2b6b98b 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1377,6 +1377,48 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
DenseMap<Register, unsigned> &InstrIdxForVirtReg,
Register ResultReg) const;
+ /// Description of an FP accumulation-chain link for the machine combiner's
+ /// FP accumulation chain reassociation (MachineCombinerPattern::FMA_CHAIN).
+ struct FMAChainLinkInfo {
+ /// Opcode of the add merging two sub-chains.
+ unsigned AddOpc;
+ /// Opcode of the plain multiply starting a new sub-chain for a fused
+ /// (FMA) link, or 0 for an unfused (FADD) link, where the new sub-chain
+ /// starts at the link's increment operand.
+ unsigned MulOpc;
+ /// Operand index of the accumulator register for a fused link, or -1 if
+ /// the accumulator is not a register (e.g. a folded memory operand),
+ /// which ends the chain. Ignored for unfused links, where the
+ /// accumulator is whichever source operand is defined by the previous
+ /// link.
+ int AccOpIdx;
+ };
+
+ /// Describe \P MI as an FP accumulation-chain link, or return std::nullopt
+ /// if \P MI is not a supported accumulation instruction. Reassociation is
+ /// only performed when the instruction carries the reassoc and nsz flags and
+ /// cannot raise FP exceptions; the generic driver checks that separately.
+ virtual std::optional<FMAChainLinkInfo>
+ getFMAChainLinkInfo(const MachineInstr &MI) const {
+ return std::nullopt;
+ }
+
+ /// Find a chain of FP accumulation instructions ending at \P Root that can
+ /// be split into two shorter chains for increased ILP. Only done at
+ /// aggressive optimization levels when not reducing register pressure and
+ /// not optimizing for size.
+ bool getFMAChainPatterns(MachineInstr &Root,
+ SmallVectorImpl<unsigned> &Patterns,
+ bool DoRegPressureReduce) const;
+
+ /// Rewrite the FP accumulation chain ending at \P Root as two shorter
+ /// chains combined by a single add.
+ void
+ reassociateFMAChain(MachineInstr &Root,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) const;
+
/// Return the inverse operation opcode if it exists for \P Opcode (e.g. add
/// for sub and vice versa).
virtual std::optional<unsigned> getInverseOpcode(unsigned Opcode) const {
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index ec2a9efa7e09c..ede208ae46a82 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -585,6 +585,8 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
// Only used when VerifyPatternOrder is enabled.
[[maybe_unused]] long PrevLatencyDiff = std::numeric_limits<long>::max();
+ [[maybe_unused]] CombinerObjective PrevObjective =
+ CombinerObjective::Default;
for (const auto P : Patterns) {
SmallVector<MachineInstr *, 16> InsInstrs;
@@ -612,15 +614,21 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
// Check that the difference between original and new latency is
// decreasing for later patterns. This helps to discover sub-optimal
- // pattern orderings.
+ // pattern orderings. Patterns with different objectives are not
+ // comparable: a pattern that must reduce the depth (e.g. splitting an
+ // accumulation chain) may save less latency than a fusion pattern that
+ // is only tried as a fallback after it.
if (VerifyPatternOrder && TSchedModel.hasInstrSchedModelOrItineraries()) {
auto [NewRootLatency, RootLatency] = getLatenciesForInstrSequences(
MI, InsInstrs, DelInstrs, TraceEnsemble->getTrace(MBB));
long CurrentLatencyDiff = ((long)RootLatency) - ((long)NewRootLatency);
- assert(CurrentLatencyDiff <= PrevLatencyDiff &&
+ CombinerObjective Objective = getCombinerObjective(P);
+ assert((Objective != PrevObjective ||
+ CurrentLatencyDiff <= PrevLatencyDiff) &&
"Current pattern is expected to be better than the previous "
"pattern.");
PrevLatencyDiff = CurrentLatencyDiff;
+ PrevObjective = Objective;
}
if (IncrementalUpdate && LastUpdate != BlockIter) {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 92fc628e888e5..c4f9f71a9cebc 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/BinaryFormat/Dwarf.h"
@@ -1137,6 +1138,268 @@ void TargetInstrInfo::reduceAccumulatorTree(
RegistersToReduce = std::move(NewRegs);
}
+// Return true if \p MO is a plain virtual register operand, i.e. one without a
+// subregister index, which can be renamed with setReg.
+static bool isPlainVirtRegOperand(const MachineOperand &MO) {
+ return MO.isReg() && MO.getReg().isVirtual() && !MO.getSubReg();
+}
+
+// Collect the FP accumulation chain ending at Root. On success Chain holds
+// the links above Root, leaf first. Returns false if the chain is too short,
+// if Root is not the last link of the chain, if the links do not all
+// accumulate the same data type, or if a link of the upper half of the chain
+// cannot be re-emitted at Root (a folded load moved across a store).
+static bool collectFMAChain(const TargetInstrInfo &TII, MachineInstr &Root,
+ SmallVectorImpl<MachineInstr *> &Chain) {
+ MachineBasicBlock &MBB = *Root.getParent();
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+ // The transform reorders the additions and de-contracts one FMA into a
+ // plain multiply. The latter rounds the product separately and can turn a
+ // -0.0 result into +0.0, so both the reassoc and nsz flags are required, as
+ // for the other FP reassociations of the machine combiner. Constrained FP
+ // instructions, which may raise FP exceptions, are left alone.
+ auto GetInfo = [&](const MachineInstr &MI)
+ -> std::optional<TargetInstrInfo::FMAChainLinkInfo> {
+ if (!MI.getFlag(MachineInstr::MIFlag::FmReassoc) ||
+ !MI.getFlag(MachineInstr::MIFlag::FmNsz) || MI.mayRaiseFPException())
+ return std::nullopt;
+ return TII.getFMAChainLinkInfo(MI);
+ };
+
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> RootInfo = GetInfo(Root);
+ if (!RootInfo)
+ return false;
+
+ // Return the link of the chain defining \p MO, or nullptr if the definition
+ // of \p MO cannot be a link: it must be a chainable instruction of the same
+ // block accumulating the same data type, and \p MO must be its only use.
+ auto GetLinkDef = [&](const MachineOperand &MO) -> MachineInstr * {
+ if (!isPlainVirtRegOperand(MO))
+ return nullptr;
+ Register R = MO.getReg();
+ MachineInstr *Def = MRI.getUniqueVRegDef(R);
+ if (!Def || Def->getParent() != &MBB)
+ return nullptr;
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> DInfo = GetInfo(*Def);
+ if (!DInfo || DInfo->AddOpc != RootInfo->AddOpc)
+ return nullptr;
+ if (!MRI.hasOneNonDBGUse(R))
+ return nullptr;
+ return Def;
+ };
+
+ // Resolve the accumulator operand of a link: for a fused link it is given
+ // by the target, for an unfused link it is the source operand defined by
+ // the preceding link of the chain. Returns -1 if the link has no chainable
+ // predecessor, which makes it the leaf of the chain.
+ auto ResolveAcc = [&](const MachineInstr &MI,
+ const TargetInstrInfo::FMAChainLinkInfo &Info) -> int {
+ if (Info.MulOpc != 0) {
+ if (Info.AccOpIdx < 0 || !GetLinkDef(MI.getOperand(Info.AccOpIdx)))
+ return -1;
+ return Info.AccOpIdx;
+ }
+ // Both source operands of an unfused link must be renamable: the
+ // increment operand becomes the accumulator of the next link when the
+ // link is dropped.
+ if (!isPlainVirtRegOperand(MI.getOperand(1)) ||
+ !isPlainVirtRegOperand(MI.getOperand(2)))
+ return -1;
+ for (unsigned Op : {1u, 2u})
+ if (GetLinkDef(MI.getOperand(Op)))
+ return Op;
+ return -1;
+ };
+
+ // Root must be the last link of the chain. The pass visits instructions
+ // top-down, so splitting is only done from the chain end; shorter prefixes
+ // are not worth splitting on their own. Root is the last link unless its
+ // single user continues the chain with Root's result as accumulator.
+ Register RootReg = Root.getOperand(0).getReg();
+ if (!RootReg.isVirtual() || Root.getOperand(0).getSubReg())
+ return false;
+ if (MRI.hasOneNonDBGUse(RootReg)) {
+ MachineInstr &User = *MRI.use_instr_nodbg_begin(RootReg);
+ if (User.getParent() == &MBB) {
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> UInfo = GetInfo(User);
+ if (UInfo && UInfo->AddOpc == RootInfo->AddOpc) {
+ int UAccIdx = ResolveAcc(User, *UInfo);
+ if (UAccIdx >= 0 && User.getOperand(UAccIdx).getReg() == RootReg)
+ return false;
+ }
+ }
+ }
+
+ // Walk the chain from Root towards the leaf.
+ MachineInstr *Link = &Root;
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> LInfo = RootInfo;
+ for (int AccIdx = ResolveAcc(*Link, *LInfo); AccIdx >= 0;
+ AccIdx = ResolveAcc(*Link, *LInfo)) {
+ Link = MRI.getUniqueVRegDef(Link->getOperand(AccIdx).getReg());
+ LInfo = GetInfo(*Link);
+ assert(LInfo && "chain predecessor must be chainable");
+ Chain.push_back(Link);
+ }
+ std::reverse(Chain.begin(), Chain.end());
+ // Need at least two links per sub-chain for the split to reduce the depth.
+ if (Chain.size() < 3)
+ return false;
+
+ // The links of the upper half of the chain other than Root are re-emitted
+ // right before Root. A link with a folded load must not be moved across a
+ // store, call or ordered memory access. The chain links are in program
+ // order, so it suffices to scan the block between the first moved link and
+ // Root once.
+ unsigned Split = (Chain.size() + 1) / 2;
+ SmallPtrSet<const MachineInstr *, 8> Moved(Chain.begin() + Split,
+ Chain.end());
+ bool MovesLoad = false;
+ for (MachineInstr &MI :
+ llvm::make_range(Chain[Split]->getIterator(), Root.getIterator())) {
+ if (Moved.contains(&MI)) {
+ if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects())
+ return false;
+ MovesLoad |= MI.mayLoad() && !MI.isDereferenceableInvariantLoad();
+ continue;
+ }
+ if (MovesLoad && (MI.mayStore() || MI.isCall() ||
+ MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()))
+ return false;
+ }
+ return true;
+}
+
+bool TargetInstrInfo::getFMAChainPatterns(MachineInstr &Root,
+ SmallVectorImpl<unsigned> &Patterns,
+ bool DoRegPressureReduce) const {
+ // The split adds an instruction and a second live accumulator. Restrict it
+ // to aggressive optimization, as for the other targets doing FMA
+ // reassociation, and do not apply it under register pressure or when
+ // optimizing for size.
+ const MachineFunction &MF = *Root.getMF();
+ if (DoRegPressureReduce ||
+ MF.getTarget().getOptLevel() != CodeGenOptLevel::Aggressive ||
+ MF.getFunction().hasOptSize())
+ return false;
+ SmallVector<MachineInstr *, 16> Chain;
+ if (!collectFMAChain(*this, Root, Chain))
+ return false;
+ Patterns.push_back(MachineCombinerPattern::FMA_CHAIN);
+ return true;
+}
+
+// Rewrite the FP accumulation chain ending at Root as two shorter chains
+// whose results are combined with a single add, halving the latency of the
+// chain:
+// L0 = FMA X, M1 (leaf of the chain)
+// L1 = FMA L0, M2
+// ...
+// Root = FMA Ln-2, Mn
+// -->
+// L0..Ls-1 unchanged
+// Us = FMUL Ms
+// Us+1 = FMA Us, Ms+1
+// ...
+// Root = FADD Ls-1, Un-1
+void TargetInstrInfo::reassociateFMAChain(
+ MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
+ MachineFunction *MF = Root.getMF();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+
+ SmallVector<MachineInstr *, 16> Chain;
+ bool Ok = collectFMAChain(*this, Root, Chain);
+ assert(Ok && "must have a chain");
+ (void)Ok;
+
+ // Full chain, leaf first, Root last.
+ Chain.push_back(&Root);
+ unsigned N = Chain.size();
+ unsigned S = N / 2;
+ assert(S >= 1 && N - S >= 2 && "bad split point");
+
+ std::optional<FMAChainLinkInfo> RootInfo = getFMAChainLinkInfo(Root);
+ assert(RootInfo && "root must be chainable");
+
+ uint32_t Flags = ~0u;
+ for (MachineInstr *Link : llvm::drop_begin(Chain, S))
+ Flags &= Link->getFlags();
+
+ const TargetRegisterClass *RC = MRI.getRegClass(Root.getOperand(0).getReg());
+ Register UpperAcc;
+ // Whether UpperAcc is a freshly created single-use register, as opposed to
+ // the reused increment of a dropped unfused link.
+ bool FreshAcc = false;
+ for (auto [I, Link] : llvm::enumerate(Chain)) {
+ if (I < S)
+ continue;
+ std::optional<FMAChainLinkInfo> LInfo = getFMAChainLinkInfo(*Link);
+ assert(LInfo && "chain link must be chainable");
+ // For an unfused link the accumulator is the source operand defined by
+ // the previous link of the chain.
+ int LAccIdx = LInfo->AccOpIdx;
+ if (LInfo->MulOpc == 0) {
+ Register PrevRes = Chain[I - 1]->getOperand(0).getReg();
+ LAccIdx = Link->getOperand(1).getReg() == PrevRes ? 1 : 2;
+ }
+ DelInstrs.push_back(Link);
+ // The link is re-emitted at Root (for a dropped unfused link, its
+ // increment is read there by the next link instead), which extends the
+ // live ranges of its source registers other than the accumulator past any
+ // intervening kill: clear their kill flags.
+ for (const MachineOperand &MO : Link->uses())
+ if (MO.isReg() && MO.getReg().isVirtual() &&
+ MO.getOperandNo() != unsigned(LAccIdx))
+ MRI.clearKillFlags(MO.getReg());
+ if (I == S && LInfo->MulOpc == 0) {
+ // Unfused link: the new sub-chain starts at the increment operand; the
+ // link itself is dropped.
+ unsigned IncIdx = LAccIdx == 1 ? 2 : 1;
+ UpperAcc = Link->getOperand(IncIdx).getReg();
+ FreshAcc = false;
+ continue;
+ }
+ Register NewDst = MRI.createVirtualRegister(RC);
+ MachineInstr *NewInstr = MF->CloneMachineInstr(Link);
+ if (I == S) {
+ // The first link of the new chain is a plain multiply; its accumulator
+ // input stays in the lower half of the chain.
+ assert(LAccIdx >= 0 && "fused link must have a register accumulator");
+ for (unsigned Op = 0, E = NewInstr->getNumOperands(); Op != E; ++Op)
+ NewInstr->untieRegOperand(Op);
+ NewInstr->removeOperand(LAccIdx);
+ NewInstr->setDesc(get(LInfo->MulOpc));
+ } else {
+ assert(LAccIdx >= 0 && "chain link must have a register accumulator");
+ NewInstr->getOperand(LAccIdx).setReg(UpperAcc);
+ NewInstr->getOperand(LAccIdx).setIsKill(FreshAcc ||
+ MRI.hasOneNonDBGUse(UpperAcc));
+ }
+ NewInstr->getOperand(0).setReg(NewDst);
+ NewInstr->setFlags(Flags);
+ InstrIdxForVirtReg.insert(std::make_pair(NewDst, InsInstrs.size()));
+ InsInstrs.push_back(NewInstr);
+ UpperAcc = NewDst;
+ FreshAcc = true;
+ }
+
+ // Combine both chains into the root's destination register.
+ Register LowerRes = Chain[S - 1]->getOperand(0).getReg();
+ Register DstReg = Root.getOperand(0).getReg();
+ MachineInstr *Add =
+ BuildMI(*MF, Root.getDebugLoc(), get(RootInfo->AddOpc), DstReg)
+ .addReg(LowerRes, getKillRegState(true))
+ .addReg(UpperAcc, getKillRegState(true));
+ Add->setFlags(Flags);
+ // The add computes the same value as Root; keep its debug instruction
+ // number.
+ if (unsigned OldRootNum = Root.peekDebugInstrNum())
+ Add->setDebugInstrNum(OldRootNum);
+ InsInstrs.push_back(Add);
+}
+
// The concept of the reassociation pass is that these operations can benefit
// from this kind of transformation:
//
@@ -1179,6 +1442,10 @@ bool TargetInstrInfo::getMachineCombinerPatterns(
if (getAccumulatorReassociationPatterns(Root, Patterns))
return true;
+ // Reassociate FP accumulation chains to expose ILP.
+ if (getFMAChainPatterns(Root, Patterns, DoRegPressureReduce))
+ return true;
+
return false;
}
@@ -1191,6 +1458,7 @@ CombinerObjective
TargetInstrInfo::getCombinerObjective(unsigned Pattern) const {
switch (Pattern) {
case MachineCombinerPattern::ACC_CHAIN:
+ case MachineCombinerPattern::FMA_CHAIN:
return CombinerObjective::MustReduceDepth;
default:
return CombinerObjective::Default;
@@ -1596,6 +1864,9 @@ void TargetInstrInfo::genAlternativeCodeSequence(
break;
}
+ case MachineCombinerPattern::FMA_CHAIN:
+ reassociateFMAChain(Root, InsInstrs, DelInstrs, InstIdxForVirtReg);
+ break;
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 872d07049ed5c..71b172b8d2ebb 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -7886,6 +7886,41 @@ static bool getMaddPatterns(MachineInstr &Root,
return Found;
}
+std::optional<TargetInstrInfo::FMAChainLinkInfo>
+AArch64InstrInfo::getFMAChainLinkInfo(const MachineInstr &MI) const {
+ FMAChainLinkInfo Info;
+ switch (MI.getOpcode()) {
+ default:
+ return std::nullopt;
+#define AARCH64_FMA_CHAIN(FMA, ADD, MUL, ACC) \
+ case AArch64::FMA: \
+ Info = {AArch64::ADD, MUL, ACC}; \
+ break;
+ // Scalar fused multiply-add: Rd = Rn * Rm + Ra.
+ AARCH64_FMA_CHAIN(FMADDSrrr, FADDSrr, AArch64::FMULSrr, 3)
+ AARCH64_FMA_CHAIN(FMADDDrrr, FADDDrr, AArch64::FMULDrr, 3)
+ // Vector fused multiply-accumulate: Vd += Vn * Vm.
+ AARCH64_FMA_CHAIN(FMLAv2f32, FADDv2f32, AArch64::FMULv2f32, 1)
+ AARCH64_FMA_CHAIN(FMLAv4f32, FADDv4f32, AArch64::FMULv4f32, 1)
+ AARCH64_FMA_CHAIN(FMLAv2f64, FADDv2f64, AArch64::FMULv2f64, 1)
+ AARCH64_FMA_CHAIN(FMLAv2i32_indexed, FADDv2f32, AArch64::FMULv2i32_indexed,
+ 1)
+ AARCH64_FMA_CHAIN(FMLAv4i32_indexed, FADDv4f32, AArch64::FMULv4i32_indexed,
+ 1)
+ AARCH64_FMA_CHAIN(FMLAv2i64_indexed, FADDv2f64, AArch64::FMULv2i64_indexed,
+ 1)
+ // Unfused FP add: the accumulator is whichever source operand is defined
+ // by the previous link of the chain.
+ AARCH64_FMA_CHAIN(FADDSrr, FADDSrr, 0, -1)
+ AARCH64_FMA_CHAIN(FADDDrr, FADDDrr, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv2f32, FADDv2f32, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv4f32, FADDv4f32, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv2f64, FADDv2f64, 0, -1)
+#undef AARCH64_FMA_CHAIN
+ }
+ return Info;
+}
+
bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
switch (Opcode) {
default:
@@ -8725,6 +8760,13 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
// Integer patterns
if (getMaddPatterns(Root, Patterns))
return true;
+
+ // The FMA chain reassociation must be tried before the FMUL+FADD fusion
+ // patterns: fusing the chain end would otherwise always win and the chain
+ // would never be split. The fusion patterns are kept as fallbacks for the
+ // case that the split does not reduce the critical path.
+ bool FoundFMAChain = getFMAChainPatterns(Root, Patterns, DoRegPressureReduce);
+
// Floating point patterns
if (getFMULPatterns(Root, Patterns))
return true;
@@ -8741,6 +8783,9 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
if (getLoadPatterns(Root, Patterns))
return true;
+ if (FoundFMAChain)
+ return true;
+
return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
DoRegPressureReduce);
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 8d913f111b44c..82b93b2e64534 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -482,6 +482,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// into a destination register.
bool isAccumulationOpcode(unsigned Opcode) const override;
+ std::optional<FMAChainLinkInfo>
+ getFMAChainLinkInfo(const MachineInstr &MI) const override;
+
/// Returns an opcode which defines the accumulator used by \P Opcode.
unsigned getAccumulationStartOpcode(unsigned Opcode) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index ee47963aa041e..bc12577ea4e36 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2800,6 +2800,30 @@ CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
}
}
+std::optional<TargetInstrInfo::FMAChainLinkInfo>
+RISCVInstrInfo::getFMAChainLinkInfo(const MachineInstr &MI) const {
+ FMAChainLinkInfo Info;
+ switch (MI.getOpcode()) {
+ default:
+ return std::nullopt;
+ // Scalar fused multiply-add: rd = rs1 * rs2 + rs3; the accumulator is the
+ // last source operand and the trailing immediate is the rounding mode.
+ case RISCV::FMADD_S:
+ Info = {RISCV::FADD_S, RISCV::FMUL_S, 3};
+ break;
+ case RISCV::FMADD_D:
+ Info = {RISCV::FADD_D, RISCV::FMUL_D, 3};
+ break;
+ case RISCV::FMADD_H:
+ Info = {RISCV::FADD_H, RISCV::FMUL_H, 3};
+ break;
+ case RISCV::FMADD_Q:
+ Info = {RISCV::FADD_Q, RISCV::FMUL_Q, 3};
+ break;
+ }
+ return Info;
+}
+
bool RISCVInstrInfo::getMachineCombinerPatterns(
MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
bool DoRegPressureReduce) const {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 559f6f4cb6069..2db3c710d53a3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -298,6 +298,9 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
SmallVectorImpl<unsigned> &Patterns,
bool DoRegPressureReduce) const override;
+ std::optional<FMAChainLinkInfo>
+ getFMAChainLinkInfo(const MachineInstr &MI) const override;
+
void
finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern,
SmallVectorImpl<MachineInstr *> &InsInstrs) const override;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 11a37efd28948..2179fd7732665 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10916,6 +10916,86 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
}
}
+// Return the VMUL/VADD opcodes matching the given VFMADD 231-form opcode for
+// FMA chain reassociation. Returns false for anything that is not a
+// supported unmasked VFMADD.
+static bool getFMAChainMulAddOpcodes(unsigned Opc231, unsigned &MulRR,
+ unsigned &MulRM, unsigned &AddRR) {
+ switch (Opc231) {
+ default:
+ return false;
+#define X86_FMA_CHAIN_OPS(FMA231, MUL, MULM, ADD) \
+ case X86::FMA231: \
+ MulRR = X86::MUL; \
+ MulRM = X86::MULM; \
+ AddRR = X86::ADD; \
+ return true;
+ X86_FMA_CHAIN_OPS(VFMADD231PDr, VMULPDrr, VMULPDrm, VADDPDrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDm, VMULPDrr, VMULPDrm, VADDPDrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDYr, VMULPDYrr, VMULPDYrm, VADDPDYrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDYm, VMULPDYrr, VMULPDYrm, VADDPDYrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZ128r, VMULPDZ128rr, VMULPDZ128rm,
+ VADDPDZ128rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZ128m, VMULPDZ128rr, VMULPDZ128rm,
+ VADDPDZ128rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZ256r, VMULPDZ256rr, VMULPDZ256rm,
+ VADDPDZ256rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZ256m, VMULPDZ256rr, VMULPDZ256rm,
+ VADDPDZ256rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZr, VMULPDZrr, VMULPDZrm, VADDPDZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PDZm, VMULPDZrr, VMULPDZrm, VADDPDZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSr, VMULPSrr, VMULPSrm, VADDPSrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSm, VMULPSrr, VMULPSrm, VADDPSrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSYr, VMULPSYrr, VMULPSYrm, VADDPSYrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSYm, VMULPSYrr, VMULPSYrm, VADDPSYrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZ128r, VMULPSZ128rr, VMULPSZ128rm,
+ VADDPSZ128rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZ128m, VMULPSZ128rr, VMULPSZ128rm,
+ VADDPSZ128rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZ256r, VMULPSZ256rr, VMULPSZ256rm,
+ VADDPSZ256rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZ256m, VMULPSZ256rr, VMULPSZ256rm,
+ VADDPSZ256rr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZr, VMULPSZrr, VMULPSZrm, VADDPSZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231PSZm, VMULPSZrr, VMULPSZrm, VADDPSZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SDr, VMULSDrr, VMULSDrm, VADDSDrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SDm, VMULSDrr, VMULSDrm, VADDSDrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SDZr, VMULSDZrr, VMULSDZrm, VADDSDZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SDZm, VMULSDZrr, VMULSDZrm, VADDSDZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SSr, VMULSSrr, VMULSSrm, VADDSSrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SSm, VMULSSrr, VMULSSrm, VADDSSrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SSZr, VMULSSZrr, VMULSSZrm, VADDSSZrr)
+ X86_FMA_CHAIN_OPS(VFMADD231SSZm, VMULSSZrr, VMULSSZrm, VADDSSZrr)
+#undef X86_FMA_CHAIN_OPS
+ }
+}
+
+std::optional<TargetInstrInfo::FMAChainLinkInfo>
+X86InstrInfo::getFMAChainLinkInfo(const MachineInstr &MI) const {
+ const X86InstrFMA3Group *G =
+ getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags);
+ if (!G || G->isKMasked() || G->isIntrinsic())
+ return std::nullopt;
+ unsigned MulRR, MulRM, AddRR;
+ if (!getFMAChainMulAddOpcodes(G->get231Opcode(), MulRR, MulRM, AddRR))
+ return std::nullopt;
+ FMAChainLinkInfo Info;
+ Info.AddOpc = AddRR;
+ unsigned Opc = MI.getOpcode();
+ if (Opc == G->get231Opcode())
+ Info.AccOpIdx = 1;
+ else if (Opc == G->get132Opcode())
+ Info.AccOpIdx = 2;
+ else
+ // For the 213 memory form the folded load is the accumulator.
+ Info.AccOpIdx = MI.mayLoad() ? -1 : 3;
+ // The plain multiply starting a new sub-chain is a memory form iff a
+ // multiplicand of MI is a folded load, which is the case exactly for the
+ // memory forms of the 231 and 132 forms.
+ Info.MulOpc = MI.mayLoad() && Info.AccOpIdx != -1 ? MulRM : MulRR;
+ return Info;
+}
+
bool X86InstrInfo::getMachineCombinerPatterns(
MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
bool DoRegPressureReduce) const {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index b6fac55373d05..068c58d293578 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -696,6 +696,9 @@ class X86InstrInfo final : public X86GenInstrInfo {
SmallVectorImpl<MachineInstr *> &DelInstrs,
DenseMap<Register, unsigned> &InstrIdxForVirtReg) const override;
+ std::optional<FMAChainLinkInfo>
+ getFMAChainLinkInfo(const MachineInstr &MI) const override;
+
/// When calculate the latency of the root instruction, accumulate the
/// latency of the sequence to the root latency.
/// \param Root - Instruction that could be combined with one of its operands
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-fma-chain.ll b/llvm/test/CodeGen/AArch64/machine-combiner-fma-chain.ll
new file mode 100644
index 0000000000000..c1a819b6965a7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-fma-chain.ll
@@ -0,0 +1,353 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O3 -mtriple=aarch64 -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; A long chain of FMA accumulations is reassociated into two shorter chains
+; combined by a single add to expose more ILP.
+
+define double @fma_chain_f64(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp d7, d16, [x0]
+; CHECK-NEXT: ldp d19, d20, [x0, #32]
+; CHECK-NEXT: ldp d17, d18, [x0, #16]
+; CHECK-NEXT: fmadd d0, d7, d1, d0
+; CHECK-NEXT: fmul d1, d18, d4
+; CHECK-NEXT: fmadd d1, d19, d5, d1
+; CHECK-NEXT: fmadd d0, d16, d2, d0
+; CHECK-NEXT: fmadd d1, d20, d6, d1
+; CHECK-NEXT: fmadd d0, d17, d3, d0
+; CHECK-NEXT: fadd d0, d0, d1
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc nsz double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call reassoc nsz double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call reassoc nsz double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc nsz double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+; A chain of unfused FADDs over FMULs is split as well; the FMUL+FADD fusion
+; happens in the machine combiner itself.
+define void @fadd_chain_loop(ptr %py, ptr %pa, i64 %n, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fadd_chain_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .p2align 5, , 16
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldr d6, [x0]
+; CHECK-NEXT: ldp d7, d16, [x1]
+; CHECK-NEXT: ldp d19, d20, [x1, #32]
+; CHECK-NEXT: add x8, x8, #1
+; CHECK-NEXT: ldp d17, d18, [x1, #16]
+; CHECK-NEXT: fmadd d6, d7, d0, d6
+; CHECK-NEXT: fmul d7, d20, d5
+; CHECK-NEXT: fmadd d6, d16, d1, d6
+; CHECK-NEXT: fmul d16, d18, d3
+; CHECK-NEXT: fmadd d16, d19, d4, d16
+; CHECK-NEXT: fmadd d6, d17, d2, d6
+; CHECK-NEXT: fadd d7, d16, d7
+; CHECK-NEXT: fadd d6, d6, d7
+; CHECK-NEXT: str d6, [x0]
+; CHECK-NEXT: cmp x8, x2
+; CHECK-NEXT: b.lo .LBB1_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inext, %loop ]
+ %y = load double, ptr %py, align 8
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %m0 = fmul reassoc nsz contract double %a0, %x0
+ %s0 = fadd reassoc nsz contract double %y, %m0
+ %m1 = fmul reassoc nsz contract double %a1, %x1
+ %s1 = fadd reassoc nsz contract double %s0, %m1
+ %m2 = fmul reassoc nsz contract double %a2, %x2
+ %s2 = fadd reassoc nsz contract double %s1, %m2
+ %m3 = fmul reassoc nsz contract double %a3, %x3
+ %s3 = fadd reassoc nsz contract double %s2, %m3
+ %m4 = fmul reassoc nsz contract double %a4, %x4
+ %s4 = fadd reassoc nsz contract double %s3, %m4
+ %m5 = fmul reassoc nsz contract double %a5, %x5
+ %s5 = fadd reassoc nsz contract double %s4, %m5
+ store double %s5, ptr %py, align 8
+ %inext = add nuw i64 %i, 1
+ %cmp = icmp ult i64 %inext, %n
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret void
+}
+
+; Without the reassoc and nsz flags the chain must be preserved.
+define double @fma_chain_no_reassoc(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_no_reassoc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp d7, d16, [x0]
+; CHECK-NEXT: ldp d17, d18, [x0, #16]
+; CHECK-NEXT: ldp d19, d20, [x0, #32]
+; CHECK-NEXT: fmadd d0, d7, d1, d0
+; CHECK-NEXT: fmadd d0, d16, d2, d0
+; CHECK-NEXT: fmadd d0, d17, d3, d0
+; CHECK-NEXT: fmadd d0, d18, d4, d0
+; CHECK-NEXT: fmadd d0, d19, d5, d0
+; CHECK-NEXT: fmadd d0, d20, d6, d0
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call contract double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call contract double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call contract double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call contract double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call contract double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call contract double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+; A chain of two FMAs is too short to split.
+define double @fma_chain_short(double %y, ptr %pa, double %x0, double %x1) {
+; CHECK-LABEL: fma_chain_short:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp d3, d4, [x0]
+; CHECK-NEXT: fmadd d0, d3, d1, d0
+; CHECK-NEXT: fmadd d0, d4, d2, d0
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ ret double %s1
+}
+
+; The multiplicand of the last link arrives late, so splitting the chain does
+; not shorten the critical path; the FMUL+FADD fusion must still happen.
+define double @fadd_chain_late_operand(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5, double %z) {
+; CHECK-LABEL: fadd_chain_late_operand:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fdiv d6, d7, d6
+; CHECK-NEXT: ldp d16, d17, [x0]
+; CHECK-NEXT: ldp d18, d19, [x0, #16]
+; CHECK-NEXT: fmadd d0, d16, d1, d0
+; CHECK-NEXT: ldp d20, d21, [x0, #32]
+; CHECK-NEXT: fmadd d0, d17, d2, d0
+; CHECK-NEXT: fmadd d0, d18, d3, d0
+; CHECK-NEXT: fmadd d0, d19, d4, d0
+; CHECK-NEXT: fmadd d0, d20, d5, d0
+; CHECK-NEXT: fdiv d6, d6, d5
+; CHECK-NEXT: fdiv d6, d6, d4
+; CHECK-NEXT: fdiv d6, d6, d3
+; CHECK-NEXT: fmadd d0, d21, d6, d0
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %d0 = fdiv double %z, %x5
+ %d1 = fdiv double %d0, %x4
+ %d2 = fdiv double %d1, %x3
+ %d3 = fdiv double %d2, %x2
+ %m0 = fmul reassoc nsz contract double %a0, %x0
+ %s0 = fadd reassoc nsz contract double %y, %m0
+ %m1 = fmul reassoc nsz contract double %a1, %x1
+ %s1 = fadd reassoc nsz contract double %s0, %m1
+ %m2 = fmul reassoc nsz contract double %a2, %x2
+ %s2 = fadd reassoc nsz contract double %s1, %m2
+ %m3 = fmul reassoc nsz contract double %a3, %x3
+ %s3 = fadd reassoc nsz contract double %s2, %m3
+ %m4 = fmul reassoc nsz contract double %a4, %x4
+ %s4 = fadd reassoc nsz contract double %s3, %m4
+ %m5 = fmul reassoc nsz contract double %a5, %d3
+ %s5 = fadd reassoc nsz contract double %s4, %m5
+ ret double %s5
+}
+
+; Vector FMLA chain.
+define <2 x double> @fma_chain_v2f64(<2 x double> %y, ptr %pa, <2 x double> %x0, <2 x double> %x1, <2 x double> %x2, <2 x double> %x3, <2 x double> %x4, <2 x double> %x5) {
+; CHECK-LABEL: fma_chain_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q7, q16, [x0]
+; CHECK-NEXT: ldp q19, q20, [x0, #64]
+; CHECK-NEXT: ldp q17, q18, [x0, #32]
+; CHECK-NEXT: fmla v0.2d, v1.2d, v7.2d
+; CHECK-NEXT: fmul v1.2d, v4.2d, v18.2d
+; CHECK-NEXT: fmla v1.2d, v5.2d, v19.2d
+; CHECK-NEXT: fmla v0.2d, v2.2d, v16.2d
+; CHECK-NEXT: fmla v1.2d, v6.2d, v20.2d
+; CHECK-NEXT: fmla v0.2d, v3.2d, v17.2d
+; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ret
+entry:
+ %a0 = load <2 x double>, ptr %pa, align 16
+ %pa1 = getelementptr <2 x double>, ptr %pa, i64 1
+ %a1 = load <2 x double>, ptr %pa1, align 16
+ %pa2 = getelementptr <2 x double>, ptr %pa, i64 2
+ %a2 = load <2 x double>, ptr %pa2, align 16
+ %pa3 = getelementptr <2 x double>, ptr %pa, i64 3
+ %a3 = load <2 x double>, ptr %pa3, align 16
+ %pa4 = getelementptr <2 x double>, ptr %pa, i64 4
+ %a4 = load <2 x double>, ptr %pa4, align 16
+ %pa5 = getelementptr <2 x double>, ptr %pa, i64 5
+ %a5 = load <2 x double>, ptr %pa5, align 16
+ %s0 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> %x0, <2 x double> %y)
+ %s1 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a1, <2 x double> %x1, <2 x double> %s0)
+ %s2 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a2, <2 x double> %x2, <2 x double> %s1)
+ %s3 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a3, <2 x double> %x3, <2 x double> %s2)
+ %s4 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a4, <2 x double> %x4, <2 x double> %s3)
+ %s5 = call reassoc nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a5, <2 x double> %x5, <2 x double> %s4)
+ ret <2 x double> %s5
+}
+
+; A loop-carried accumulation chain.
+define double @fma_chain_loop_carried(ptr %pa, ptr %pb, i64 %n) {
+; CHECK-LABEL: fma_chain_loop_carried:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: add x9, x0, #24
+; CHECK-NEXT: add x10, x1, #24
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: .p2align 5, , 16
+; CHECK-NEXT: .LBB6_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldp d1, d2, [x9, #-24]
+; CHECK-NEXT: ldp d3, d4, [x10, #-24]
+; CHECK-NEXT: ldp d17, d18, [x9, #8]
+; CHECK-NEXT: ldp d19, d20, [x10, #8]
+; CHECK-NEXT: ldp d5, d6, [x9, #-8]
+; CHECK-NEXT: ldp d7, d16, [x10, #-8]
+; CHECK-NEXT: add x8, x8, #6
+; CHECK-NEXT: add x9, x9, #48
+; CHECK-NEXT: add x10, x10, #48
+; CHECK-NEXT: fmadd d0, d1, d3, d0
+; CHECK-NEXT: fmul d1, d6, d16
+; CHECK-NEXT: fmadd d1, d17, d19, d1
+; CHECK-NEXT: fmadd d0, d2, d4, d0
+; CHECK-NEXT: fmadd d1, d18, d20, d1
+; CHECK-NEXT: fmadd d0, d5, d7, d0
+; CHECK-NEXT: fadd d0, d0, d1
+; CHECK-NEXT: cmp x8, x2
+; CHECK-NEXT: b.lo .LBB6_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inext, %loop ]
+ %acc = phi double [ 0.0, %entry ], [ %s5, %loop ]
+ %p = getelementptr double, ptr %pa, i64 %i
+ %q = getelementptr double, ptr %pb, i64 %i
+ %a0 = load double, ptr %p, align 8
+ %b0 = load double, ptr %q, align 8
+ %p1 = getelementptr double, ptr %p, i64 1
+ %q1 = getelementptr double, ptr %q, i64 1
+ %a1 = load double, ptr %p1, align 8
+ %b1 = load double, ptr %q1, align 8
+ %p2 = getelementptr double, ptr %p, i64 2
+ %q2 = getelementptr double, ptr %q, i64 2
+ %a2 = load double, ptr %p2, align 8
+ %b2 = load double, ptr %q2, align 8
+ %p3 = getelementptr double, ptr %p, i64 3
+ %q3 = getelementptr double, ptr %q, i64 3
+ %a3 = load double, ptr %p3, align 8
+ %b3 = load double, ptr %q3, align 8
+ %p4 = getelementptr double, ptr %p, i64 4
+ %q4 = getelementptr double, ptr %q, i64 4
+ %a4 = load double, ptr %p4, align 8
+ %b4 = load double, ptr %q4, align 8
+ %p5 = getelementptr double, ptr %p, i64 5
+ %q5 = getelementptr double, ptr %q, i64 5
+ %a5 = load double, ptr %p5, align 8
+ %b5 = load double, ptr %q5, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %b0, double %acc)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %b1, double %s0)
+ %s2 = call reassoc nsz double @llvm.fma.f64(double %a2, double %b2, double %s1)
+ %s3 = call reassoc nsz double @llvm.fma.f64(double %a3, double %b3, double %s2)
+ %s4 = call reassoc nsz double @llvm.fma.f64(double %a4, double %b4, double %s3)
+ %s5 = call reassoc nsz double @llvm.fma.f64(double %a5, double %b5, double %s4)
+ %inext = add nuw i64 %i, 6
+ %cmp = icmp ult i64 %inext, %n
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret double %s5
+}
+
+; With the reassoc flag but without nsz the chain must be preserved:
+; de-contracting an FMA into a plain multiply can change the sign of a zero.
+define double @fma_chain_no_nsz(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_no_nsz:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp d7, d16, [x0]
+; CHECK-NEXT: ldp d17, d18, [x0, #16]
+; CHECK-NEXT: ldp d19, d20, [x0, #32]
+; CHECK-NEXT: fmadd d0, d7, d1, d0
+; CHECK-NEXT: fmadd d0, d16, d2, d0
+; CHECK-NEXT: fmadd d0, d17, d3, d0
+; CHECK-NEXT: fmadd d0, d18, d4, d0
+; CHECK-NEXT: fmadd d0, d19, d5, d0
+; CHECK-NEXT: fmadd d0, d20, d6, d0
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call reassoc double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call reassoc double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call reassoc double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+declare double @llvm.fma.f64(double, double, double)
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
diff --git a/llvm/test/CodeGen/RISCV/machine-combiner-fma-chain.ll b/llvm/test/CodeGen/RISCV/machine-combiner-fma-chain.ll
new file mode 100644
index 0000000000000..0a3ff8c23f5bf
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/machine-combiner-fma-chain.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O3 -mtriple=riscv64 -mattr=+d < %s | FileCheck %s
+
+; A long chain of FMA accumulations is reassociated into two shorter chains
+; combined by a single add to expose more ILP.
+
+define double @fma_chain_f64(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_f64:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: fld ft0, 0(a0)
+; CHECK-NEXT: fld ft1, 24(a0)
+; CHECK-NEXT: fld ft2, 8(a0)
+; CHECK-NEXT: fld ft3, 16(a0)
+; CHECK-NEXT: fmadd.d fa1, ft0, fa1, fa0
+; CHECK-NEXT: fmul.d fa4, ft1, fa4
+; CHECK-NEXT: fld fa0, 32(a0)
+; CHECK-NEXT: fmadd.d fa2, ft2, fa2, fa1
+; CHECK-NEXT: fld fa1, 40(a0)
+; CHECK-NEXT: fmadd.d fa5, fa0, fa5, fa4
+; CHECK-NEXT: fmadd.d fa4, ft3, fa3, fa2
+; CHECK-NEXT: fmadd.d fa5, fa1, fa6, fa5
+; CHECK-NEXT: fadd.d fa0, fa4, fa5
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %m0 = fmul reassoc nsz contract double %a0, %x0
+ %s0 = fadd reassoc nsz contract double %y, %m0
+ %m1 = fmul reassoc nsz contract double %a1, %x1
+ %s1 = fadd reassoc nsz contract double %s0, %m1
+ %m2 = fmul reassoc nsz contract double %a2, %x2
+ %s2 = fadd reassoc nsz contract double %s1, %m2
+ %m3 = fmul reassoc nsz contract double %a3, %x3
+ %s3 = fadd reassoc nsz contract double %s2, %m3
+ %m4 = fmul reassoc nsz contract double %a4, %x4
+ %s4 = fadd reassoc nsz contract double %s3, %m4
+ %m5 = fmul reassoc nsz contract double %a5, %x5
+ %s5 = fadd reassoc nsz contract double %s4, %m5
+ ret double %s5
+}
+
+; Without the nsz flag the chain must be preserved.
+define double @fma_chain_no_nsz(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_no_nsz:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: fld ft0, 0(a0)
+; CHECK-NEXT: fld ft1, 8(a0)
+; CHECK-NEXT: fld ft2, 16(a0)
+; CHECK-NEXT: fld ft3, 24(a0)
+; CHECK-NEXT: fmadd.d fa1, ft0, fa1, fa0
+; CHECK-NEXT: fmadd.d fa2, ft1, fa2, fa1
+; CHECK-NEXT: fmadd.d fa3, ft2, fa3, fa2
+; CHECK-NEXT: fmadd.d fa4, ft3, fa4, fa3
+; CHECK-NEXT: fld fa3, 32(a0)
+; CHECK-NEXT: fld fa2, 40(a0)
+; CHECK-NEXT: fmadd.d fa5, fa3, fa5, fa4
+; CHECK-NEXT: fmadd.d fa0, fa2, fa6, fa5
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %m0 = fmul reassoc contract double %a0, %x0
+ %s0 = fadd reassoc contract double %y, %m0
+ %m1 = fmul reassoc contract double %a1, %x1
+ %s1 = fadd reassoc contract double %s0, %m1
+ %m2 = fmul reassoc contract double %a2, %x2
+ %s2 = fadd reassoc contract double %s1, %m2
+ %m3 = fmul reassoc contract double %a3, %x3
+ %s3 = fadd reassoc contract double %s2, %m3
+ %m4 = fmul reassoc contract double %a4, %x4
+ %s4 = fadd reassoc contract double %s3, %m4
+ %m5 = fmul reassoc contract double %a5, %x5
+ %s5 = fadd reassoc contract double %s4, %m5
+ ret double %s5
+}
+
+; A chain of two FMAs is too short to split.
+define double @fma_chain_short(double %y, ptr %pa, double %x0, double %x1) {
+; CHECK-LABEL: fma_chain_short:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: fld fa5, 0(a0)
+; CHECK-NEXT: fld fa4, 8(a0)
+; CHECK-NEXT: fmadd.d fa5, fa5, fa1, fa0
+; CHECK-NEXT: fmadd.d fa0, fa4, fa2, fa5
+; CHECK-NEXT: ret
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %m0 = fmul reassoc nsz contract double %a0, %x0
+ %s0 = fadd reassoc nsz contract double %y, %m0
+ %m1 = fmul reassoc nsz contract double %a1, %x1
+ %s1 = fadd reassoc nsz contract double %s0, %m1
+ ret double %s1
+}
diff --git a/llvm/test/CodeGen/X86/machine-combiner-fma-chain.ll b/llvm/test/CodeGen/X86/machine-combiner-fma-chain.ll
new file mode 100644
index 0000000000000..d23e4cfa4b5ea
--- /dev/null
+++ b/llvm/test/CodeGen/X86/machine-combiner-fma-chain.ll
@@ -0,0 +1,329 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O3 -mcpu=haswell < %s -mtriple=x86_64 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: llc -O3 -mcpu=skylake-avx512 < %s -mtriple=x86_64 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; A long chain of FMA accumulations is reassociated into two shorter chains
+; combined by a single add to expose more ILP.
+
+define <4 x double> @fma_chain_v4f64(ptr %py, ptr %pa, <4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3, <4 x double> %x4, <4 x double> %x5) {
+; CHECK-LABEL: fma_chain_v4f64:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovupd (%rsi), %ymm6
+; CHECK-NEXT: vfmadd213pd {{.*#+}} ymm6 = (ymm0 * ymm6) + mem
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm6 = (ymm1 * mem) + ymm6
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm6 = (ymm2 * mem) + ymm6
+; CHECK-NEXT: vmulpd 96(%rsi), %ymm3, %ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm4 * mem) + ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm5 * mem) + ymm0
+; CHECK-NEXT: vaddpd %ymm0, %ymm6, %ymm0
+; CHECK-NEXT: retq
+entry:
+ %y = load <4 x double>, ptr %py, align 8
+ %a0 = load <4 x double>, ptr %pa, align 8
+ %pa1 = getelementptr <4 x double>, ptr %pa, i64 1
+ %a1 = load <4 x double>, ptr %pa1, align 8
+ %pa2 = getelementptr <4 x double>, ptr %pa, i64 2
+ %a2 = load <4 x double>, ptr %pa2, align 8
+ %pa3 = getelementptr <4 x double>, ptr %pa, i64 3
+ %a3 = load <4 x double>, ptr %pa3, align 8
+ %pa4 = getelementptr <4 x double>, ptr %pa, i64 4
+ %a4 = load <4 x double>, ptr %pa4, align 8
+ %pa5 = getelementptr <4 x double>, ptr %pa, i64 5
+ %a5 = load <4 x double>, ptr %pa5, align 8
+ %m0 = fmul reassoc nsz contract <4 x double> %a0, %x0
+ %s0 = fadd reassoc nsz contract <4 x double> %y, %m0
+ %m1 = fmul reassoc nsz contract <4 x double> %a1, %x1
+ %s1 = fadd reassoc nsz contract <4 x double> %s0, %m1
+ %m2 = fmul reassoc nsz contract <4 x double> %a2, %x2
+ %s2 = fadd reassoc nsz contract <4 x double> %s1, %m2
+ %m3 = fmul reassoc nsz contract <4 x double> %a3, %x3
+ %s3 = fadd reassoc nsz contract <4 x double> %s2, %m3
+ %m4 = fmul reassoc nsz contract <4 x double> %a4, %x4
+ %s4 = fadd reassoc nsz contract <4 x double> %s3, %m4
+ %m5 = fmul reassoc nsz contract <4 x double> %a5, %x5
+ %s5 = fadd reassoc nsz contract <4 x double> %s4, %m5
+ ret <4 x double> %s5
+}
+
+define double @fma_chain_f64(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_f64:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm2 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm3 * mem) + xmm0
+; CHECK-NEXT: vmulsd 24(%rdi), %xmm4, %xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm5 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm6 * mem) + xmm1
+; CHECK-NEXT: vaddsd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %m0 = fmul reassoc nsz contract double %a0, %x0
+ %s0 = fadd reassoc nsz contract double %y, %m0
+ %m1 = fmul reassoc nsz contract double %a1, %x1
+ %s1 = fadd reassoc nsz contract double %s0, %m1
+ %m2 = fmul reassoc nsz contract double %a2, %x2
+ %s2 = fadd reassoc nsz contract double %s1, %m2
+ %m3 = fmul reassoc nsz contract double %a3, %x3
+ %s3 = fadd reassoc nsz contract double %s2, %m3
+ %m4 = fmul reassoc nsz contract double %a4, %x4
+ %s4 = fadd reassoc nsz contract double %s3, %m4
+ %m5 = fmul reassoc nsz contract double %a5, %x5
+ %s5 = fadd reassoc nsz contract double %s4, %m5
+ ret double %s5
+}
+
+; Without the reassoc and nsz flags the chain must be preserved.
+define <4 x double> @fma_chain_no_reassoc(ptr %py, ptr %pa, <4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3, <4 x double> %x4, <4 x double> %x5) {
+; CHECK-LABEL: fma_chain_no_reassoc:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovupd (%rsi), %ymm6
+; CHECK-NEXT: vfmadd213pd {{.*#+}} ymm0 = (ymm6 * ymm0) + mem
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm2 * mem) + ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm3 * mem) + ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm4 * mem) + ymm0
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm5 * mem) + ymm0
+; CHECK-NEXT: retq
+entry:
+ %y = load <4 x double>, ptr %py, align 8
+ %a0 = load <4 x double>, ptr %pa, align 8
+ %pa1 = getelementptr <4 x double>, ptr %pa, i64 1
+ %a1 = load <4 x double>, ptr %pa1, align 8
+ %pa2 = getelementptr <4 x double>, ptr %pa, i64 2
+ %a2 = load <4 x double>, ptr %pa2, align 8
+ %pa3 = getelementptr <4 x double>, ptr %pa, i64 3
+ %a3 = load <4 x double>, ptr %pa3, align 8
+ %pa4 = getelementptr <4 x double>, ptr %pa, i64 4
+ %a4 = load <4 x double>, ptr %pa4, align 8
+ %pa5 = getelementptr <4 x double>, ptr %pa, i64 5
+ %a5 = load <4 x double>, ptr %pa5, align 8
+ %m0 = fmul contract <4 x double> %a0, %x0
+ %s0 = fadd contract <4 x double> %y, %m0
+ %m1 = fmul contract <4 x double> %a1, %x1
+ %s1 = fadd contract <4 x double> %s0, %m1
+ %m2 = fmul contract <4 x double> %a2, %x2
+ %s2 = fadd contract <4 x double> %s1, %m2
+ %m3 = fmul contract <4 x double> %a3, %x3
+ %s3 = fadd contract <4 x double> %s2, %m3
+ %m4 = fmul contract <4 x double> %a4, %x4
+ %s4 = fadd contract <4 x double> %s3, %m4
+ %m5 = fmul contract <4 x double> %a5, %x5
+ %s5 = fadd contract <4 x double> %s4, %m5
+ ret <4 x double> %s5
+}
+
+; A chain of two FMAs is too short to split.
+define <4 x double> @fma_chain_short(ptr %py, ptr %pa, <4 x double> %x0, <4 x double> %x1) {
+; CHECK-LABEL: fma_chain_short:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovupd (%rsi), %ymm2
+; CHECK-NEXT: vfmadd213pd {{.*#+}} ymm0 = (ymm2 * ymm0) + mem
+; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0
+; CHECK-NEXT: retq
+entry:
+ %y = load <4 x double>, ptr %py, align 8
+ %a0 = load <4 x double>, ptr %pa, align 8
+ %pa1 = getelementptr <4 x double>, ptr %pa, i64 1
+ %a1 = load <4 x double>, ptr %pa1, align 8
+ %m0 = fmul reassoc nsz contract <4 x double> %a0, %x0
+ %s0 = fadd reassoc nsz contract <4 x double> %y, %m0
+ %m1 = fmul reassoc nsz contract <4 x double> %a1, %x1
+ %s1 = fadd reassoc nsz contract <4 x double> %s0, %m1
+ ret <4 x double> %s1
+}
+
+; A store that may alias the load folded into a link of the upper half of the
+; chain sits between that link and the chain end: the load must not be moved
+; past the store, so the chain is not split.
+define double @fma_chain_store_between(double %y, ptr %pa, ptr %pq, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_store_between:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd132sd {{.*#+}} xmm1 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm2 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm3 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm4 * mem) + xmm1
+; CHECK-NEXT: vmovsd %xmm0, (%rsi)
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm5 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm6 * mem) + xmm1
+; CHECK-NEXT: vmovapd %xmm1, %xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc nsz double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call reassoc nsz double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ store double %y, ptr %pq, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s4 = call reassoc nsz double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc nsz double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+; A store before the first link of the upper half does not block the split.
+define double @fma_chain_store_before_upper(double %y, ptr %pa, ptr %pq, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_store_before_upper:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd132sd {{.*#+}} xmm1 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm2 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm3 * mem) + xmm1
+; CHECK-NEXT: vmovsd %xmm0, (%rsi)
+; CHECK-NEXT: vmulsd 24(%rdi), %xmm4, %xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm5 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm6 * mem) + xmm0
+; CHECK-NEXT: vaddsd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc nsz double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ store double %y, ptr %pq, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s3 = call reassoc nsz double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call reassoc nsz double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc nsz double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+; The chain end has a second user besides the continuing FMA, so the chain
+; ending there is split even though the last FMA accumulates into it.
+define double @fma_chain_root_multi_use(double %y, ptr %pa, ptr %pq, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5, double %x6) {
+; CHECK-LABEL: fma_chain_root_multi_use:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm2 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm3 * mem) + xmm0
+; CHECK-NEXT: vmulsd 24(%rdi), %xmm4, %xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm5 * mem) + xmm1
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm1 = (xmm6 * mem) + xmm1
+; CHECK-NEXT: vaddsd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT: vmovsd %xmm0, (%rsi)
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm7 * mem) + xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call reassoc nsz double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc nsz double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc nsz double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call reassoc nsz double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call reassoc nsz double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc nsz double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ store double %s5, ptr %pq, align 8
+ %pa6 = getelementptr double, ptr %pa, i64 6
+ %a6 = load double, ptr %pa6, align 8
+ %s6 = call reassoc nsz double @llvm.fma.f64(double %a6, double %x6, double %s5)
+ ret double %s6
+}
+
+; With the reassoc flag but without nsz the chain must be preserved:
+; de-contracting an FMA into a plain multiply can change the sign of a zero.
+define double @fma_chain_no_nsz(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) {
+; CHECK-LABEL: fma_chain_no_nsz:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm2 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm3 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm4 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm5 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm6 * mem) + xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call reassoc double @llvm.fma.f64(double %a0, double %x0, double %y)
+ %s1 = call reassoc double @llvm.fma.f64(double %a1, double %x1, double %s0)
+ %s2 = call reassoc double @llvm.fma.f64(double %a2, double %x2, double %s1)
+ %s3 = call reassoc double @llvm.fma.f64(double %a3, double %x3, double %s2)
+ %s4 = call reassoc double @llvm.fma.f64(double %a4, double %x4, double %s3)
+ %s5 = call reassoc double @llvm.fma.f64(double %a5, double %x5, double %s4)
+ ret double %s5
+}
+
+; Constrained FP operations may raise exceptions and are left alone.
+define double @fma_chain_strictfp(double %y, ptr %pa, double %x0, double %x1, double %x2, double %x3, double %x4, double %x5) #0 {
+; CHECK-LABEL: fma_chain_strictfp:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm2 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm3 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm4 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm5 * mem) + xmm0
+; CHECK-NEXT: vfmadd231sd {{.*#+}} xmm0 = (xmm6 * mem) + xmm0
+; CHECK-NEXT: retq
+entry:
+ %a0 = load double, ptr %pa, align 8
+ %pa1 = getelementptr double, ptr %pa, i64 1
+ %a1 = load double, ptr %pa1, align 8
+ %pa2 = getelementptr double, ptr %pa, i64 2
+ %a2 = load double, ptr %pa2, align 8
+ %pa3 = getelementptr double, ptr %pa, i64 3
+ %a3 = load double, ptr %pa3, align 8
+ %pa4 = getelementptr double, ptr %pa, i64 4
+ %a4 = load double, ptr %pa4, align 8
+ %pa5 = getelementptr double, ptr %pa, i64 5
+ %a5 = load double, ptr %pa5, align 8
+ %s0 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a0, double %x0, double %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %s1 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a1, double %x1, double %s0, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %s2 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a2, double %x2, double %s1, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %s3 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a3, double %x3, double %s2, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %s4 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a4, double %x4, double %s3, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %s5 = call reassoc nsz double @llvm.experimental.constrained.fma.f64(double %a5, double %x5, double %s4, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ ret double %s5
+}
+
+declare double @llvm.fma.f64(double, double, double)
+declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
+
+attributes #0 = { strictfp }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX2: {{.*}}
+; AVX512: {{.*}}
More information about the llvm-commits
mailing list