[llvm] [MachineCombiner]Reassociate FP accumulation chains to expose ILP (PR #222676)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 07:48:17 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Add a target-independent machine combiner pattern, FMA_CHAIN, that
rewrites a chain of FP multiply-accumulations (fused FMA, or unfused
FADD over FMUL) ending at a root into two shorter sub-chains combined
by a single add, roughly halving the chain's critical-path latency. The
split requires the reassoc and nsz flags and no FP exceptions, and is
applied only when it reduces the critical path.
Targets opt in via the new TargetInstrInfo::getFMAChainLinkInfo hook,
which reports the plain-multiply and add opcodes and the accumulator
operand index of a chain link. Implemented for X86 (VFMADD, all widths,
register and memory-folded forms), AArch64 (FMADD, FMLA, and unfused
FADD), and RISC-V (scalar FMADD).
Based on PPC support, existing in LLVM, but extending it for other
targets.
Assisted-by: Cursor
---
Patch is 65.84 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222676.diff
13 Files Affected:
- (modified) llvm/include/llvm/CodeGen/MachineCombinerPattern.h (+3)
- (modified) llvm/include/llvm/CodeGen/TargetInstrInfo.h (+42)
- (modified) llvm/lib/CodeGen/MachineCombiner.cpp (+10-2)
- (modified) llvm/lib/CodeGen/TargetInstrInfo.cpp (+271)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.cpp (+45)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.h (+3)
- (modified) llvm/lib/Target/RISCV/RISCVInstrInfo.cpp (+24)
- (modified) llvm/lib/Target/RISCV/RISCVInstrInfo.h (+3)
- (modified) llvm/lib/Target/X86/X86InstrInfo.cpp (+80)
- (modified) llvm/lib/Target/X86/X86InstrInfo.h (+3)
- (added) llvm/test/CodeGen/AArch64/machine-combiner-fma-chain.ll (+353)
- (added) llvm/test/CodeGen/RISCV/machine-combiner-fma-chain.ll (+113)
- (added) llvm/test/CodeGen/X86/machine-combiner-fma-chain.ll (+329)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
index 25fce679323ee..b37ae84f8f77f 100644
--- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
+++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
@@ -33,6 +33,9 @@ enum MachineCombinerPattern : unsigned {
REASSOC_XA_BY,
REASSOC_XA_YB,
ACC_CHAIN,
+ // Reassociation of an FP accumulation (FMA) chain into two shorter chains
+ // combined by a single add, to expose more ILP.
+ FMA_CHAIN,
TARGET_PATTERN_START
};
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index f27261e7be1aa..2e662c2b6b98b 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1377,6 +1377,48 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
DenseMap<Register, unsigned> &InstrIdxForVirtReg,
Register ResultReg) const;
+ /// Description of an FP accumulation-chain link for the machine combiner's
+ /// FP accumulation chain reassociation (MachineCombinerPattern::FMA_CHAIN).
+ struct FMAChainLinkInfo {
+ /// Opcode of the add merging two sub-chains.
+ unsigned AddOpc;
+ /// Opcode of the plain multiply starting a new sub-chain for a fused
+ /// (FMA) link, or 0 for an unfused (FADD) link, where the new sub-chain
+ /// starts at the link's increment operand.
+ unsigned MulOpc;
+ /// Operand index of the accumulator register for a fused link, or -1 if
+ /// the accumulator is not a register (e.g. a folded memory operand),
+ /// which ends the chain. Ignored for unfused links, where the
+ /// accumulator is whichever source operand is defined by the previous
+ /// link.
+ int AccOpIdx;
+ };
+
+ /// Describe \P MI as an FP accumulation-chain link, or return std::nullopt
+ /// if \P MI is not a supported accumulation instruction. Reassociation is
+ /// only performed when the instruction carries the reassoc and nsz flags and
+ /// cannot raise FP exceptions; the generic driver checks that separately.
+ virtual std::optional<FMAChainLinkInfo>
+ getFMAChainLinkInfo(const MachineInstr &MI) const {
+ return std::nullopt;
+ }
+
+ /// Find a chain of FP accumulation instructions ending at \P Root that can
+ /// be split into two shorter chains for increased ILP. Only done at
+ /// aggressive optimization levels when not reducing register pressure and
+ /// not optimizing for size.
+ bool getFMAChainPatterns(MachineInstr &Root,
+ SmallVectorImpl<unsigned> &Patterns,
+ bool DoRegPressureReduce) const;
+
+ /// Rewrite the FP accumulation chain ending at \P Root as two shorter
+ /// chains combined by a single add.
+ void
+ reassociateFMAChain(MachineInstr &Root,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) const;
+
/// Return the inverse operation opcode if it exists for \P Opcode (e.g. add
/// for sub and vice versa).
virtual std::optional<unsigned> getInverseOpcode(unsigned Opcode) const {
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index ec2a9efa7e09c..ede208ae46a82 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -585,6 +585,8 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
// Only used when VerifyPatternOrder is enabled.
[[maybe_unused]] long PrevLatencyDiff = std::numeric_limits<long>::max();
+ [[maybe_unused]] CombinerObjective PrevObjective =
+ CombinerObjective::Default;
for (const auto P : Patterns) {
SmallVector<MachineInstr *, 16> InsInstrs;
@@ -612,15 +614,21 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
// Check that the difference between original and new latency is
// decreasing for later patterns. This helps to discover sub-optimal
- // pattern orderings.
+ // pattern orderings. Patterns with different objectives are not
+ // comparable: a pattern that must reduce the depth (e.g. splitting an
+ // accumulation chain) may save less latency than a fusion pattern that
+ // is only tried as a fallback after it.
if (VerifyPatternOrder && TSchedModel.hasInstrSchedModelOrItineraries()) {
auto [NewRootLatency, RootLatency] = getLatenciesForInstrSequences(
MI, InsInstrs, DelInstrs, TraceEnsemble->getTrace(MBB));
long CurrentLatencyDiff = ((long)RootLatency) - ((long)NewRootLatency);
- assert(CurrentLatencyDiff <= PrevLatencyDiff &&
+ CombinerObjective Objective = getCombinerObjective(P);
+ assert((Objective != PrevObjective ||
+ CurrentLatencyDiff <= PrevLatencyDiff) &&
"Current pattern is expected to be better than the previous "
"pattern.");
PrevLatencyDiff = CurrentLatencyDiff;
+ PrevObjective = Objective;
}
if (IncrementalUpdate && LastUpdate != BlockIter) {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 92fc628e888e5..c4f9f71a9cebc 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/BinaryFormat/Dwarf.h"
@@ -1137,6 +1138,268 @@ void TargetInstrInfo::reduceAccumulatorTree(
RegistersToReduce = std::move(NewRegs);
}
+// Return true if \p MO is a plain virtual register operand, i.e. one without a
+// subregister index, which can be renamed with setReg.
+static bool isPlainVirtRegOperand(const MachineOperand &MO) {
+ return MO.isReg() && MO.getReg().isVirtual() && !MO.getSubReg();
+}
+
+// Collect the FP accumulation chain ending at Root. On success Chain holds
+// the links above Root, leaf first. Returns false if the chain is too short,
+// if Root is not the last link of the chain, if the links do not all
+// accumulate the same data type, or if a link of the upper half of the chain
+// cannot be re-emitted at Root (a folded load moved across a store).
+static bool collectFMAChain(const TargetInstrInfo &TII, MachineInstr &Root,
+ SmallVectorImpl<MachineInstr *> &Chain) {
+ MachineBasicBlock &MBB = *Root.getParent();
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+ // The transform reorders the additions and de-contracts one FMA into a
+ // plain multiply. The latter rounds the product separately and can turn a
+ // -0.0 result into +0.0, so both the reassoc and nsz flags are required, as
+ // for the other FP reassociations of the machine combiner. Constrained FP
+ // instructions, which may raise FP exceptions, are left alone.
+ auto GetInfo = [&](const MachineInstr &MI)
+ -> std::optional<TargetInstrInfo::FMAChainLinkInfo> {
+ if (!MI.getFlag(MachineInstr::MIFlag::FmReassoc) ||
+ !MI.getFlag(MachineInstr::MIFlag::FmNsz) || MI.mayRaiseFPException())
+ return std::nullopt;
+ return TII.getFMAChainLinkInfo(MI);
+ };
+
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> RootInfo = GetInfo(Root);
+ if (!RootInfo)
+ return false;
+
+ // Return the link of the chain defining \p MO, or nullptr if the definition
+ // of \p MO cannot be a link: it must be a chainable instruction of the same
+ // block accumulating the same data type, and \p MO must be its only use.
+ auto GetLinkDef = [&](const MachineOperand &MO) -> MachineInstr * {
+ if (!isPlainVirtRegOperand(MO))
+ return nullptr;
+ Register R = MO.getReg();
+ MachineInstr *Def = MRI.getUniqueVRegDef(R);
+ if (!Def || Def->getParent() != &MBB)
+ return nullptr;
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> DInfo = GetInfo(*Def);
+ if (!DInfo || DInfo->AddOpc != RootInfo->AddOpc)
+ return nullptr;
+ if (!MRI.hasOneNonDBGUse(R))
+ return nullptr;
+ return Def;
+ };
+
+ // Resolve the accumulator operand of a link: for a fused link it is given
+ // by the target, for an unfused link it is the source operand defined by
+ // the preceding link of the chain. Returns -1 if the link has no chainable
+ // predecessor, which makes it the leaf of the chain.
+ auto ResolveAcc = [&](const MachineInstr &MI,
+ const TargetInstrInfo::FMAChainLinkInfo &Info) -> int {
+ if (Info.MulOpc != 0) {
+ if (Info.AccOpIdx < 0 || !GetLinkDef(MI.getOperand(Info.AccOpIdx)))
+ return -1;
+ return Info.AccOpIdx;
+ }
+ // Both source operands of an unfused link must be renamable: the
+ // increment operand becomes the accumulator of the next link when the
+ // link is dropped.
+ if (!isPlainVirtRegOperand(MI.getOperand(1)) ||
+ !isPlainVirtRegOperand(MI.getOperand(2)))
+ return -1;
+ for (unsigned Op : {1u, 2u})
+ if (GetLinkDef(MI.getOperand(Op)))
+ return Op;
+ return -1;
+ };
+
+ // Root must be the last link of the chain. The pass visits instructions
+ // top-down, so splitting is only done from the chain end; shorter prefixes
+ // are not worth splitting on their own. Root is the last link unless its
+ // single user continues the chain with Root's result as accumulator.
+ Register RootReg = Root.getOperand(0).getReg();
+ if (!RootReg.isVirtual() || Root.getOperand(0).getSubReg())
+ return false;
+ if (MRI.hasOneNonDBGUse(RootReg)) {
+ MachineInstr &User = *MRI.use_instr_nodbg_begin(RootReg);
+ if (User.getParent() == &MBB) {
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> UInfo = GetInfo(User);
+ if (UInfo && UInfo->AddOpc == RootInfo->AddOpc) {
+ int UAccIdx = ResolveAcc(User, *UInfo);
+ if (UAccIdx >= 0 && User.getOperand(UAccIdx).getReg() == RootReg)
+ return false;
+ }
+ }
+ }
+
+ // Walk the chain from Root towards the leaf.
+ MachineInstr *Link = &Root;
+ std::optional<TargetInstrInfo::FMAChainLinkInfo> LInfo = RootInfo;
+ for (int AccIdx = ResolveAcc(*Link, *LInfo); AccIdx >= 0;
+ AccIdx = ResolveAcc(*Link, *LInfo)) {
+ Link = MRI.getUniqueVRegDef(Link->getOperand(AccIdx).getReg());
+ LInfo = GetInfo(*Link);
+ assert(LInfo && "chain predecessor must be chainable");
+ Chain.push_back(Link);
+ }
+ std::reverse(Chain.begin(), Chain.end());
+ // Need at least two links per sub-chain for the split to reduce the depth.
+ if (Chain.size() < 3)
+ return false;
+
+ // The links of the upper half of the chain other than Root are re-emitted
+ // right before Root. A link with a folded load must not be moved across a
+ // store, call or ordered memory access. The chain links are in program
+ // order, so it suffices to scan the block between the first moved link and
+ // Root once.
+ unsigned Split = (Chain.size() + 1) / 2;
+ SmallPtrSet<const MachineInstr *, 8> Moved(Chain.begin() + Split,
+ Chain.end());
+ bool MovesLoad = false;
+ for (MachineInstr &MI :
+ llvm::make_range(Chain[Split]->getIterator(), Root.getIterator())) {
+ if (Moved.contains(&MI)) {
+ if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects())
+ return false;
+ MovesLoad |= MI.mayLoad() && !MI.isDereferenceableInvariantLoad();
+ continue;
+ }
+ if (MovesLoad && (MI.mayStore() || MI.isCall() ||
+ MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()))
+ return false;
+ }
+ return true;
+}
+
+bool TargetInstrInfo::getFMAChainPatterns(MachineInstr &Root,
+ SmallVectorImpl<unsigned> &Patterns,
+ bool DoRegPressureReduce) const {
+ // The split adds an instruction and a second live accumulator. Restrict it
+ // to aggressive optimization, as for the other targets doing FMA
+ // reassociation, and do not apply it under register pressure or when
+ // optimizing for size.
+ const MachineFunction &MF = *Root.getMF();
+ if (DoRegPressureReduce ||
+ MF.getTarget().getOptLevel() != CodeGenOptLevel::Aggressive ||
+ MF.getFunction().hasOptSize())
+ return false;
+ SmallVector<MachineInstr *, 16> Chain;
+ if (!collectFMAChain(*this, Root, Chain))
+ return false;
+ Patterns.push_back(MachineCombinerPattern::FMA_CHAIN);
+ return true;
+}
+
+// Rewrite the FP accumulation chain ending at Root as two shorter chains
+// whose results are combined with a single add, halving the latency of the
+// chain:
+// L0 = FMA X, M1 (leaf of the chain)
+// L1 = FMA L0, M2
+// ...
+// Root = FMA Ln-2, Mn
+// -->
+// L0..Ls-1 unchanged
+// Us = FMUL Ms
+// Us+1 = FMA Us, Ms+1
+// ...
+// Root = FADD Ls-1, Un-1
+void TargetInstrInfo::reassociateFMAChain(
+ MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
+ MachineFunction *MF = Root.getMF();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+
+ SmallVector<MachineInstr *, 16> Chain;
+ bool Ok = collectFMAChain(*this, Root, Chain);
+ assert(Ok && "must have a chain");
+ (void)Ok;
+
+ // Full chain, leaf first, Root last.
+ Chain.push_back(&Root);
+ unsigned N = Chain.size();
+ unsigned S = N / 2;
+ assert(S >= 1 && N - S >= 2 && "bad split point");
+
+ std::optional<FMAChainLinkInfo> RootInfo = getFMAChainLinkInfo(Root);
+ assert(RootInfo && "root must be chainable");
+
+ uint32_t Flags = ~0u;
+ for (MachineInstr *Link : llvm::drop_begin(Chain, S))
+ Flags &= Link->getFlags();
+
+ const TargetRegisterClass *RC = MRI.getRegClass(Root.getOperand(0).getReg());
+ Register UpperAcc;
+ // Whether UpperAcc is a freshly created single-use register, as opposed to
+ // the reused increment of a dropped unfused link.
+ bool FreshAcc = false;
+ for (auto [I, Link] : llvm::enumerate(Chain)) {
+ if (I < S)
+ continue;
+ std::optional<FMAChainLinkInfo> LInfo = getFMAChainLinkInfo(*Link);
+ assert(LInfo && "chain link must be chainable");
+ // For an unfused link the accumulator is the source operand defined by
+ // the previous link of the chain.
+ int LAccIdx = LInfo->AccOpIdx;
+ if (LInfo->MulOpc == 0) {
+ Register PrevRes = Chain[I - 1]->getOperand(0).getReg();
+ LAccIdx = Link->getOperand(1).getReg() == PrevRes ? 1 : 2;
+ }
+ DelInstrs.push_back(Link);
+ // The link is re-emitted at Root (for a dropped unfused link, its
+ // increment is read there by the next link instead), which extends the
+ // live ranges of its source registers other than the accumulator past any
+ // intervening kill: clear their kill flags.
+ for (const MachineOperand &MO : Link->uses())
+ if (MO.isReg() && MO.getReg().isVirtual() &&
+ MO.getOperandNo() != unsigned(LAccIdx))
+ MRI.clearKillFlags(MO.getReg());
+ if (I == S && LInfo->MulOpc == 0) {
+ // Unfused link: the new sub-chain starts at the increment operand; the
+ // link itself is dropped.
+ unsigned IncIdx = LAccIdx == 1 ? 2 : 1;
+ UpperAcc = Link->getOperand(IncIdx).getReg();
+ FreshAcc = false;
+ continue;
+ }
+ Register NewDst = MRI.createVirtualRegister(RC);
+ MachineInstr *NewInstr = MF->CloneMachineInstr(Link);
+ if (I == S) {
+ // The first link of the new chain is a plain multiply; its accumulator
+ // input stays in the lower half of the chain.
+ assert(LAccIdx >= 0 && "fused link must have a register accumulator");
+ for (unsigned Op = 0, E = NewInstr->getNumOperands(); Op != E; ++Op)
+ NewInstr->untieRegOperand(Op);
+ NewInstr->removeOperand(LAccIdx);
+ NewInstr->setDesc(get(LInfo->MulOpc));
+ } else {
+ assert(LAccIdx >= 0 && "chain link must have a register accumulator");
+ NewInstr->getOperand(LAccIdx).setReg(UpperAcc);
+ NewInstr->getOperand(LAccIdx).setIsKill(FreshAcc ||
+ MRI.hasOneNonDBGUse(UpperAcc));
+ }
+ NewInstr->getOperand(0).setReg(NewDst);
+ NewInstr->setFlags(Flags);
+ InstrIdxForVirtReg.insert(std::make_pair(NewDst, InsInstrs.size()));
+ InsInstrs.push_back(NewInstr);
+ UpperAcc = NewDst;
+ FreshAcc = true;
+ }
+
+ // Combine both chains into the root's destination register.
+ Register LowerRes = Chain[S - 1]->getOperand(0).getReg();
+ Register DstReg = Root.getOperand(0).getReg();
+ MachineInstr *Add =
+ BuildMI(*MF, Root.getDebugLoc(), get(RootInfo->AddOpc), DstReg)
+ .addReg(LowerRes, getKillRegState(true))
+ .addReg(UpperAcc, getKillRegState(true));
+ Add->setFlags(Flags);
+ // The add computes the same value as Root; keep its debug instruction
+ // number.
+ if (unsigned OldRootNum = Root.peekDebugInstrNum())
+ Add->setDebugInstrNum(OldRootNum);
+ InsInstrs.push_back(Add);
+}
+
// The concept of the reassociation pass is that these operations can benefit
// from this kind of transformation:
//
@@ -1179,6 +1442,10 @@ bool TargetInstrInfo::getMachineCombinerPatterns(
if (getAccumulatorReassociationPatterns(Root, Patterns))
return true;
+ // Reassociate FP accumulation chains to expose ILP.
+ if (getFMAChainPatterns(Root, Patterns, DoRegPressureReduce))
+ return true;
+
return false;
}
@@ -1191,6 +1458,7 @@ CombinerObjective
TargetInstrInfo::getCombinerObjective(unsigned Pattern) const {
switch (Pattern) {
case MachineCombinerPattern::ACC_CHAIN:
+ case MachineCombinerPattern::FMA_CHAIN:
return CombinerObjective::MustReduceDepth;
default:
return CombinerObjective::Default;
@@ -1596,6 +1864,9 @@ void TargetInstrInfo::genAlternativeCodeSequence(
break;
}
+ case MachineCombinerPattern::FMA_CHAIN:
+ reassociateFMAChain(Root, InsInstrs, DelInstrs, InstIdxForVirtReg);
+ break;
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 872d07049ed5c..71b172b8d2ebb 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -7886,6 +7886,41 @@ static bool getMaddPatterns(MachineInstr &Root,
return Found;
}
+std::optional<TargetInstrInfo::FMAChainLinkInfo>
+AArch64InstrInfo::getFMAChainLinkInfo(const MachineInstr &MI) const {
+ FMAChainLinkInfo Info;
+ switch (MI.getOpcode()) {
+ default:
+ return std::nullopt;
+#define AARCH64_FMA_CHAIN(FMA, ADD, MUL, ACC) \
+ case AArch64::FMA: \
+ Info = {AArch64::ADD, MUL, ACC}; \
+ break;
+ // Scalar fused multiply-add: Rd = Rn * Rm + Ra.
+ AARCH64_FMA_CHAIN(FMADDSrrr, FADDSrr, AArch64::FMULSrr, 3)
+ AARCH64_FMA_CHAIN(FMADDDrrr, FADDDrr, AArch64::FMULDrr, 3)
+ // Vector fused multiply-accumulate: Vd += Vn * Vm.
+ AARCH64_FMA_CHAIN(FMLAv2f32, FADDv2f32, AArch64::FMULv2f32, 1)
+ AARCH64_FMA_CHAIN(FMLAv4f32, FADDv4f32, AArch64::FMULv4f32, 1)
+ AARCH64_FMA_CHAIN(FMLAv2f64, FADDv2f64, AArch64::FMULv2f64, 1)
+ AARCH64_FMA_CHAIN(FMLAv2i32_indexed, FADDv2f32, AArch64::FMULv2i32_indexed,
+ 1)
+ AARCH64_FMA_CHAIN(FMLAv4i32_indexed, FADDv4f32, AArch64::FMULv4i32_indexed,
+ 1)
+ AARCH64_FMA_CHAIN(FMLAv2i64_indexed, FADDv2f64, AArch64::FMULv2i64_indexed,
+ 1)
+ // Unfused FP add: the accumulator is whichever source operand is defined
+ // by the previous link of the chain.
+ AARCH64_FMA_CHAIN(FADDSrr, FADDSrr, 0, -1)
+ AARCH64_FMA_CHAIN(FADDDrr, FADDDrr, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv2f32, FADDv2f32, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv4f32, FADDv4f32, 0, -1)
+ AARCH64_FMA_CHAIN(FADDv2f64, FADDv2f64, 0, -1)
+#undef AARCH64_FMA_CHAIN
+ }
+ return Info;
+}
+
bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
switch (Opcode) {
default:
@@ -8725,6 +8760,13 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222676
More information about the llvm-commits
mailing list