[llvm-branch-commits] [llvm] [AMDGPU] Insert MFMA anti-hints in GCNPreRAOptimizations (PR #218075)
Syadus Sefat via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 26 08:51:31 PDT 2026
https://github.com/mssefat updated https://github.com/llvm/llvm-project/pull/218075
>From dea57bab26d05d5fdb841530e90c7bfa6f5b49fc Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 19 Aug 2026 23:51:22 -0500
Subject: [PATCH] [AMDGPU] Add MFMA anti-hints in GCNPreRAOptimizations
---
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp | 510 ++++++++
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h | 124 ++
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 19 +
.../CodeGen/AMDGPU/anti-hints-mfma-war.mir | 559 +++++++++
.../CodeGen/AMDGPU/anti-hints-mfma-waw.mir | 1060 +++++++++++++++++
.../llvm.amdgcn.mfma.anti-hints.gfx942.mir | 145 +++
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 64 +-
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 118 +-
.../AMDGPU/mfma-no-register-aliasing.ll | 7 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 60 +-
11 files changed, 2543 insertions(+), 124 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
create mode 100644 llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..a881750cf4f38 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -132,6 +132,7 @@ add_llvm_target(AMDGPUCodeGen
GCNIterativeScheduler.cpp
GCNMinRegStrategy.cpp
GCNNSAReassign.cpp
+ GCNPreRAAntiHints.cpp
GCNPreRAOptimizations.cpp
GCNPreRALongBranchReg.cpp
GCNRegPressure.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
new file mode 100644
index 0000000000000..231b1adaea871
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
@@ -0,0 +1,510 @@
+//===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert register allocation anti-hints.
+///
+//===----------------------------------------------------------------------===//
+
+#include "GCNPreRAAntiHints.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+
+using namespace llvm;
+using namespace llvm::AMDGPU;
+
+#define DEBUG_TYPE "amdgpu-anti-hints"
+
+namespace HC = llvm::AMDGPU::HazardClass;
+
+static cl::opt<std::string> AntiHintRuleSelection(
+ "amdgpu-anti-hints-rules", cl::Hidden,
+ cl::desc("Comma-separated anti-hints rules (waw, war), or all or none."),
+ cl::init("all"));
+
+namespace {
+
+// Classify the MI into a HazardClassMask.
+HazardClassMask getInstHazardClass(const MachineInstr &MI,
+ const HazardContext &Ctx) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ HazardClassMask Mask = HC::None;
+
+ if (TII.isLDSDMA(MI))
+ Mask = HC::VALU | HC::VMEM | HC::DS;
+ else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
+ Mask = HC::WMMA;
+ else if (TII.isMFMA(MI))
+ Mask = HC::MFMA;
+ else if (SIInstrInfo::isTRANS(MI))
+ Mask = HC::TRANS;
+ else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
+ Mask = HC::VALU;
+ else if (TII.isDS(MI))
+ Mask = HC::DS;
+ else if (TII.isVMEM(MI))
+ Mask = HC::VMEM;
+ else if (TII.isSMRD(MI))
+ Mask = HC::SMEM;
+ else if (TII.isEXP(MI))
+ Mask = HC::EXP;
+ else if (SIInstrInfo::isSALU(MI))
+ Mask = HC::SALU;
+
+ return Mask;
+}
+
+void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
+ const HazardContext &Ctx,
+ SmallVectorImpl<Register> &Out) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ auto Add = [&](const MachineOperand *MO) {
+ if (MO && MO->isReg() && MO->getReg().isVirtual() &&
+ Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO->getReg())))
+ Out.push_back(MO->getReg());
+ };
+ auto Named = [&](AMDGPU::OpName N) { Add(TII.getNamedOperand(MI, N)); };
+ switch (Op) {
+ case HazardOperand::None:
+ break;
+ case HazardOperand::Def:
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isDef())
+ Add(&MO);
+ break;
+ case HazardOperand::Src0:
+ Named(AMDGPU::OpName::src0);
+ break;
+ case HazardOperand::Src1:
+ Named(AMDGPU::OpName::src1);
+ break;
+ case HazardOperand::Src2:
+ Named(AMDGPU::OpName::src2);
+ break;
+ case HazardOperand::Idx:
+ Named(AMDGPU::OpName::idx);
+ break;
+ case HazardOperand::Vaddr:
+ Named(AMDGPU::OpName::vaddr);
+ break;
+ case HazardOperand::AnySrc:
+ Named(AMDGPU::OpName::src0);
+ Named(AMDGPU::OpName::src1);
+ Named(AMDGPU::OpName::src2);
+ break;
+ case HazardOperand::AnyUse:
+ for (const MachineOperand &MO : MI.uses())
+ if (MO.isReg() && MO.isUse())
+ Add(&MO);
+ break;
+ }
+}
+
+enum class MFMAHazardKind { RAW, WAW, WAR };
+
+// MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait
+// states.
+unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
+ HazardClassMask ReaderClass, const HazardContext &Ctx) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ const GCNSubtarget &ST = *Ctx.ST;
+ const int NumPasses = Ctx.SchedModel->computeInstrLatency(&MFMA);
+ const bool IsDGEMM = SIInstrInfo::isDGEMM(MFMA.getOpcode());
+ const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP);
+
+ auto GFX940NPass = [&]() -> unsigned {
+ return TII.isXDL(MFMA)
+ ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
+ : NumPasses + 2;
+ };
+ auto SMFMANPass = [&]() -> unsigned {
+ switch (NumPasses) {
+ case 2:
+ return 5;
+ case 8:
+ return 11;
+ case 16:
+ return 19;
+ default:
+ return 0;
+ }
+ };
+
+ switch (Kind) {
+ case MFMAHazardKind::RAW:
+ if (IsDGEMM) {
+ switch (NumPasses) {
+ case 4:
+ return Mem ? 9 : 6;
+ case 8:
+ case 16:
+ return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
+ default:
+ return 0;
+ }
+ }
+ return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
+
+ case MFMAHazardKind::WAW:
+ if (IsDGEMM) {
+ switch (NumPasses) {
+ case 4:
+ return 6;
+ case 8:
+ case 16:
+ return 11;
+ default:
+ return 0;
+ }
+ }
+ return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
+
+ case MFMAHazardKind::WAR:
+ switch (NumPasses) {
+ case 2:
+ return 1;
+ case 4:
+ return 3;
+ case 8:
+ return 7;
+ case 16:
+ return 15;
+ default:
+ return 15;
+ }
+ }
+ return 0;
+}
+
+unsigned mfmaWawWindow(const MachineInstr &P, const HazardContext &Ctx) {
+ return mfmaWaitStates(P, MFMAHazardKind::WAW, HC::None, Ctx);
+}
+unsigned mfmaWarWindow(const MachineInstr &P, const HazardContext &Ctx) {
+ return mfmaWaitStates(P, MFMAHazardKind::WAR, HC::None, Ctx);
+}
+
+unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
+ HazardClassMask ReaderClass,
+ const HazardContext &Ctx) {
+ return mfmaWaitStates(Producer, MFMAHazardKind::RAW, ReaderClass, Ctx);
+}
+
+bool hasMFMAHazard(const HazardContext &Ctx) {
+ return Ctx.ST->hasGFX90AInsts();
+}
+
+bool ruleSelected(StringRef Name) {
+ StringRef Selection(AntiHintRuleSelection);
+ if (Selection.equals_insensitive("all"))
+ return true;
+ if (Selection.equals_insensitive("none"))
+ return false;
+ SmallVector<StringRef, 3> Selected;
+ Selection.split(Selected, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+ return llvm::any_of(Selected, [Name](StringRef S) {
+ return S.trim().equals_insensitive(Name);
+ });
+}
+
+bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
+ return hasMFMAHazard(Ctx) && ruleSelected("waw");
+}
+
+bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
+ return hasMFMAHazard(Ctx) && ruleSelected("war");
+}
+
+bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
+ return Ctx.TII->isXDL(MI);
+}
+
+unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
+ const HazardContext &Ctx) {
+ // Explicity given window length overrides the computed one.
+ if (CT.Window.OptWindowLength &&
+ CT.Window.OptWindowLength->getNumOccurrences())
+ return *CT.Window.OptWindowLength;
+ if (CT.Window.Fn)
+ return CT.Window.Fn(MI, Ctx);
+ return CT.Window.WindowLength;
+}
+
+// Build the anti-hints rules.
+class HazardRuleSet {
+ SmallVector<HazardAntiHintRule, 0> Rules;
+
+public:
+ class RuleBuilder {
+
+ HazardRuleSet &S;
+ unsigned Idx;
+ HazardAntiHintRule &rule() const { return S.Rules[Idx]; }
+
+ public:
+ RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {}
+
+ RuleBuilder &producer(ClassMatch M, HazardOperand Op,
+ InstPredicate Predicate = nullptr) {
+ rule().Producer = {M, Op, Predicate};
+ return *this;
+ }
+
+ RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) {
+ rule().AdvanceForRawWindow = AdvanceForRawWindow;
+ return *this;
+ }
+
+ RuleBuilder &consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
+ HazardClassMask CountMask = 0,
+ ConsumerHint Hint = ConsumerHint::OneDirectional,
+ InstPredicate Predicate = nullptr) {
+ rule().Consumers.push_back({{M, Op, Predicate}, Window, CountMask, Hint});
+ return *this;
+ }
+
+ RuleBuilder &enabledIf(RulePredicate Predicate) {
+ rule().Predicate = Predicate;
+ return *this;
+ }
+ };
+
+ RuleBuilder addRule() {
+ Rules.emplace_back();
+ return RuleBuilder(*this, Rules.size() - 1);
+ }
+
+ SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); }
+};
+
+// Here the anti-hints rules are inserted.
+SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
+ using HO = HazardOperand;
+ HazardRuleSet S;
+
+ const WindowSpec MfmaWawWindow{0, nullptr, mfmaWawWindow};
+ const WindowSpec MfmaWarWindow{0, nullptr, mfmaWarWindow};
+ const ClassMatch MfmaConsumers = {HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
+ HC::EXP};
+
+ // MFMA WAW rules
+ S.addRule()
+ .enabledIf(isMFMAWAWRuleEnabled)
+ .producer({HC::MFMA}, HO::Def)
+ .rawCredit(mfmaReaderRawWindow)
+ .consumer(MfmaConsumers, HO::Def, MfmaWawWindow, HC::None,
+ ConsumerHint::OneDirectional);
+
+ // MFMA WAR rules
+ S.addRule()
+ .enabledIf(isMFMAWARRuleEnabled)
+ .producer({HC::MFMA}, HO::Src2, isXDLMFMA)
+ .rawCredit(mfmaReaderRawWindow)
+ .consumer(MfmaConsumers, HO::Def, MfmaWarWindow, HC::None,
+ ConsumerHint::OneDirectional);
+
+ return S.buildRules();
+}
+
+ArrayRef<HazardAntiHintRule> getAntiHintsRules() {
+ static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules();
+ return Rules;
+}
+
+struct AntiHintWindow {
+ SmallVector<Register, 4> Regs;
+ const MachineInstr *Producer = nullptr;
+ unsigned Len = 0;
+ unsigned Elapsed = 0;
+};
+
+using ConsumerTracking = SmallVector<AntiHintWindow, 3>;
+// One per consumer target of a rule.
+using RuleTracking = SmallVector<ConsumerTracking, 3>;
+
+class AntiHintEngine {
+ const HazardContext &Ctx;
+ ArrayRef<HazardAntiHintRule> Rules;
+
+ SmallVector<bool, 8> RuleApplies;
+ bool AnyEnabled = false;
+
+public:
+ AntiHintEngine(const HazardContext &Ctx)
+ : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx);
+ AnyEnabled |= RuleApplies[R];
+ }
+ }
+
+ void run(MachineFunction &MF) {
+ if (!AnyEnabled)
+ return;
+
+ SmallVector<RuleTracking, 8> Tracking(Rules.size());
+ for (unsigned R = 0; R < Rules.size(); ++R)
+ Tracking[R].resize(Rules[R].Consumers.size());
+
+ for (const MachineBasicBlock &MBB : MF) {
+ for (RuleTracking &RT : Tracking)
+ for (ConsumerTracking &Track : RT)
+ Track.clear();
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isMetaInstruction())
+ continue;
+ const HazardClassMask C = getInstHazardClass(MI, Ctx);
+ // Wait states this instruction contributes to an open window.
+ unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI);
+ addAntiHintsAndExpire(MI, C, WaitStates, Tracking);
+ addWindows(MI, C, Tracking);
+ }
+ }
+ }
+
+private:
+ // Check if the right class and predicate matches.
+ bool sideMatches(const HazardSide &Side, const MachineInstr &MI,
+ HazardClassMask C) {
+ return Side.Match.matches(C) &&
+ (!Side.Predicate || Side.Predicate(MI, Ctx));
+ }
+
+ // Producer phase: open a window for each consumer target.
+ void addWindows(const MachineInstr &MI, HazardClassMask C,
+ MutableArrayRef<RuleTracking> Tracking) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ if (!RuleApplies[R] || !sideMatches(Rule.Producer, MI, C))
+ continue;
+
+ SmallVector<Register, 4> Regs;
+ // Collect the producer regs.
+ collectOperandRegs(MI, Rule.Producer.Op, Ctx, Regs);
+ if (Regs.empty())
+ continue;
+ for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
+ ConsumerIdx != E; ++ConsumerIdx) {
+ unsigned Window = resolveWindow(Rule.Consumers[ConsumerIdx], MI, Ctx);
+ if (!Window)
+ continue;
+ Tracking[R][ConsumerIdx].push_back({Regs, &MI, Window, 0});
+ }
+ }
+ }
+
+ void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
+ const HazardAntiHintRule &Rule,
+ const ConsumerTarget &CT, ConsumerTracking &Track) {
+
+ if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(C))
+ return;
+ for (AntiHintWindow &Window : Track) {
+ // Determine if def generated by producer is read by the MI consumer.
+ bool ReadsProducerDef = llvm::any_of(
+ Window.Producer->all_defs(), [&](const MachineOperand &MO) {
+ return MO.getReg().isVirtual() &&
+ MI.readsVirtualRegister(MO.getReg());
+ });
+ // Advance by RAW window from the producer if that is larger than current
+ // Window.Elapsed.
+ if (ReadsProducerDef)
+ Window.Elapsed = std::max(
+ Window.Elapsed, Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx));
+ }
+ }
+
+ // Consumer phase: add anti-hints, then charge and expire open windows.
+ void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C,
+ unsigned WaitStates,
+ MutableArrayRef<RuleTracking> Tracking) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ if (!RuleApplies[R])
+ continue;
+ for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
+ ConsumerIdx != E; ++ConsumerIdx) {
+ const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx];
+ ConsumerTracking &Track = Tracking[R][ConsumerIdx];
+ if (Track.empty())
+ continue;
+
+ // Before adding the anti-hints, see if advancing by RAW window will
+ // help remove the window.
+ advanceByRawWindow(MI, C, Rule, CT, Track);
+ llvm::erase_if(Track, [](const AntiHintWindow &Window) {
+ return Window.Elapsed >= Window.Len;
+ });
+
+ // Add anti-hints if the consumer matches the instruction.
+ if (sideMatches(CT.Side, MI, C))
+ addAntiHints(CT, MI, Track);
+
+ // This instruction's own wait states count toward the next one.
+ if (!CT.CounterMask || (C & CT.CounterMask))
+ for (AntiHintWindow &Window : Track)
+ Window.Elapsed += WaitStates;
+ }
+ }
+ }
+
+ bool isCopyOf(Register Cand, Register HazardReg) const {
+ const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Cand);
+ return Def && Def->isCopy() && Def->getOperand(1).getReg() == HazardReg;
+ }
+
+ void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI,
+ const ConsumerTracking &Track) {
+ if (Track.empty())
+ return;
+ SmallVector<Register, 4> ConsumerRegs;
+ // Collect the consumer regs.
+ collectOperandRegs(MI, CT.Side.Op, Ctx, ConsumerRegs);
+ if (ConsumerRegs.empty())
+ return;
+ SlotIndex Slot = Ctx.LIS->getInstructionIndex(MI).getRegSlot();
+ auto AntiHint = [&](Register ProducerReg) {
+ if (!Ctx.LIS->hasInterval(ProducerReg))
+ return;
+ const LiveInterval &ProducerLI = Ctx.LIS->getInterval(ProducerReg);
+ if (ProducerLI.liveAt(Slot))
+ return;
+ for (Register ConsumerReg : ConsumerRegs) {
+ if (ConsumerReg == ProducerReg || isCopyOf(ConsumerReg, ProducerReg))
+ continue;
+ // Only allow anti-hints when producer and consumer reg live ranges do
+ // not overlap
+ if (Ctx.LIS->hasInterval(ConsumerReg) &&
+ ProducerLI.overlaps(Ctx.LIS->getInterval(ConsumerReg)))
+ continue;
+ Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
+ if (CT.Hint == ConsumerHint::Symmetric)
+ Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
+ LLVM_DEBUG(
+ dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
+ << (CT.Hint == ConsumerHint::Symmetric ? " <-> " : " <- ")
+ << printReg(ConsumerReg, Ctx.TRI) << " (consumer "
+ << Ctx.TII->getName(MI.getOpcode()) << ")\n");
+ }
+ };
+ for (const AntiHintWindow &Window : Track)
+ for (Register ProducerReg : Window.Regs)
+ AntiHint(ProducerReg);
+ }
+};
+
+} // namespace
+
+void AMDGPU::applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx) {
+ AntiHintEngine(Ctx).run(MF);
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
new file mode 100644
index 0000000000000..e9b0e884d549b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
@@ -0,0 +1,124 @@
+//===- GCNPreRAAntiHints.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
+#define LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+class MachineFunction;
+class MachineInstr;
+class MachineRegisterInfo;
+class LiveIntervals;
+class SIRegisterInfo;
+class SIInstrInfo;
+class GCNSubtarget;
+class TargetSchedModel;
+
+namespace AMDGPU {
+
+using HazardClassMask = uint32_t;
+namespace HazardClass {
+
+// Coherent with CoExec instruction types
+enum : HazardClassMask {
+ None = 0,
+ CTRL = 1u << 0,
+ VALU = 1u << 1,
+ TRANS = 1u << 2,
+ SALU = 1u << 3,
+ DS = 1u << 4,
+ VMEM = 1u << 5,
+ SMEM = 1u << 6,
+ WMMA = 1u << 7,
+ MFMA = 1u << 8,
+ EXP = 1u << 9,
+};
+} // namespace HazardClass
+
+enum class HazardOperand : uint8_t {
+ None,
+ Def,
+ Src0,
+ Src1,
+ Src2,
+ Idx,
+ Vaddr,
+ AnySrc,
+ AnyUse,
+};
+
+enum class ConsumerHint : uint8_t {
+ OneDirectional,
+ Symmetric,
+};
+
+struct HazardContext {
+ const SIInstrInfo *TII;
+ const SIRegisterInfo *TRI;
+ MachineRegisterInfo *MRI;
+ LiveIntervals *LIS;
+ const GCNSubtarget *ST;
+ const TargetSchedModel *SchedModel;
+};
+
+struct ClassMatch {
+ HazardClassMask AnyOf = 0;
+ bool matches(HazardClassMask Mask) const { return !AnyOf || (Mask & AnyOf); }
+};
+
+// To enable or disable the whole rule.
+using RulePredicate = bool (*)(const HazardContext &);
+
+// To gate the rule for producer or consumer.
+using InstPredicate = bool (*)(const MachineInstr &, const HazardContext &);
+
+// This helps advancing a window for the WAW/WAR case where a RAW sits between
+// the producer and the consumer.
+using AdvanceForRawWindowFn = unsigned (*)(const MachineInstr &Producer,
+ HazardClassMask ReaderClass,
+ const HazardContext &Ctx);
+
+struct HazardSide {
+ ClassMatch Match;
+ HazardOperand Op = HazardOperand::None;
+ InstPredicate Predicate = nullptr;
+};
+
+// Length of a hazard window in wait states.
+// Precedence: OptWindowLength --> Fn --> WindowLength.
+struct WindowSpec {
+ unsigned WindowLength = 0;
+ const cl::opt<unsigned> *OptWindowLength = nullptr;
+ unsigned (*Fn)(const MachineInstr &Producer,
+ const HazardContext &Ctx) = nullptr;
+};
+
+struct ConsumerTarget {
+ HazardSide Side;
+ WindowSpec Window;
+ HazardClassMask CounterMask = HazardClass::None;
+ ConsumerHint Hint = ConsumerHint::OneDirectional;
+};
+
+struct HazardAntiHintRule {
+ RulePredicate Predicate = nullptr;
+ HazardSide Producer;
+ SmallVector<ConsumerTarget, 3> Consumers;
+ AdvanceForRawWindowFn AdvanceForRawWindow = nullptr;
+};
+
+void applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx);
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index 50514e819f412..3c9c5fef86779 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -32,17 +32,27 @@
#include "GCNPreRAOptimizations.h"
#include "AMDGPU.h"
+#include "GCNPreRAAntiHints.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
using namespace llvm;
#define DEBUG_TYPE "amdgpu-pre-ra-optimizations"
+static cl::opt<bool> EnableAntiHintsForMFMARegs(
+ "amdgpu-anti-hints-for-mfma", cl::Hidden,
+ cl::desc("Add register allocation anti-hints for MFMA hazards."),
+ cl::init(true));
+
namespace {
class GCNPreRAOptimizationsImpl {
@@ -51,6 +61,7 @@ class GCNPreRAOptimizationsImpl {
const SIRegisterInfo *TRI;
MachineRegisterInfo *MRI;
LiveIntervals *LIS;
+ TargetSchedModel SchedModel;
bool processReg(Register Reg);
void hintTrue16Copy(const MachineInstr &MI);
@@ -305,6 +316,14 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
+ // Anti-hints only steer register allocation, so they do not count as a
+ // modification of the function.
+ if (EnableAntiHintsForMFMARegs) {
+ SchedModel.init(&ST);
+ AMDGPU::HazardContext HCtx{TII, TRI, MRI, LIS, &ST, &SchedModel};
+ AMDGPU::applyAntiHintRules(MF, HCtx);
+ }
+
bool Changed = false;
for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
new file mode 100644
index 0000000000000..148d2ced6399f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
@@ -0,0 +1,559 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX90A
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX942
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX950
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX90A-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX942-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX950-NOAH
+
+--- |
+ define amdgpu_kernel void @mfma_war_xdl_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_vmem() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_ds() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_nonxdl_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_nonxdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_multi_mfma() #0 { ret void }
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+---
+name: mfma_war_xdl_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 6
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 2
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 2
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_128_align2 = IMPLICIT_DEF
+ %7:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ $vgpr40_vgpr41_vgpr42_vgpr43 = COPY %7
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+ - { reg: '$vgpr17' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16, $vgpr17
+ ; AH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; AH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vgpr_32 = COPY $vgpr17
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 4096, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_vmem
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_ds
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %7
+...
+
+---
+name: mfma_war_nonxdl_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-GFX90A-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX90A: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX90A-NEXT: {{ $}}
+ ; AH-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; AH-GFX942-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX942: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX942-NEXT: {{ $}}
+ ; AH-GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX942-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; AH-GFX950-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX950: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX950-NEXT: {{ $}}
+ ; AH-GFX950-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX950-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX950-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 6
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_128_align2 = IMPLICIT_DEF
+ %7:vreg_128_align2 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ $vgpr40_vgpr41_vgpr42_vgpr43 = COPY %7
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit %8
+...
+
+---
+name: mfma_war_nonxdl_16pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-GFX90A-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX90A: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX90A-NEXT: {{ $}}
+ ; AH-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX90A-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; AH-GFX942-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX942: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX942-NEXT: {{ $}}
+ ; AH-GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX942-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; AH-GFX950-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX950: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX950-NEXT: {{ $}}
+ ; AH-GFX950-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX950-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX950-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_multi_mfma
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+ ; AH-LABEL: name: mfma_war_multi_mfma
+ ; AH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX90A-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 12
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 13
+ ; GFX90A-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX942-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 4
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 5
+ ; GFX942-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX950-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 4
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 5
+ ; GFX950-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
new file mode 100644
index 0000000000000..8404e34080096
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
@@ -0,0 +1,1060 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX90A-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX942-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX950-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX90A-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX942-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX950-NOAH
+
+--- |
+ define amdgpu_kernel void @mfma_waw_nonxdl_2pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_2pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_vmem() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_ds() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_4pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_16pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_4pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_boundary_4pass_valu() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: mfma_waw_nonxdl_2pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 4
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 3
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 3
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 %0.sub0, %0.sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_2pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_2pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; NOAH-LABEL: name: mfma_waw_xdl_2pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 4
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_dgemm_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_dgemm_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr12_vgpr13 = BUFFER_LOAD_DWORDX2_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_4pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 5
+ ; NOAH-NEXT: renamable $vgpr10_vgpr11 = BUFFER_LOAD_DWORDX2_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_64_align2 = IMPLICIT_DEF
+ %5:vreg_64_align2 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_64_align2 = BUFFER_LOAD_DWORDX2_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 9
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 9
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 %0.sub0, %0.sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_8pass_vmem
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_8pass_ds
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %7
+...
+
+---
+name: mfma_waw_dgemm_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_dgemm_8pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr26_vgpr27_vgpr28_vgpr29 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr26_vgpr27_vgpr28_vgpr29
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_8pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 10
+ ; NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_256_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_16pass
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 1
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 1
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 1
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_16pass
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 1
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_16pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr82_vgpr83_vgpr84_vgpr85 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr82_vgpr83_vgpr84_vgpr85
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 2
+ ; GFX942-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 3
+ ; GFX950-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_1024_align2 = IMPLICIT_DEF
+ %7:vreg_1024_align2 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr18_vgpr19, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %6, %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4
+...
+
+---
+name: mfma_waw_xdl_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr50_vgpr51, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %8, %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_16pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr50_vgpr51, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 1
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 1
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %8, %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_dgemm_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-LABEL: name: mfma_waw_dgemm_4pass_valu
+ ; AH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr12, $vgpr13, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_4pass_valu
+ ; NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 5
+ ; NOAH-NEXT: renamable $vgpr10_vgpr11 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr10, $vgpr11, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $sgpr0_sgpr1
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %3:vreg_64_align2 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %2, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ %5:sreg_64_xexec = V_CMP_EQ_U32_e64 %4.sub0, %4.sub1, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5
+...
+
+---
+name: mfma_waw_dgemm_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; AH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr24_vgpr25 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr24, $vgpr25, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 10
+ ; GFX950-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vreg_256_align2 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %2, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ %5:sreg_64_xexec = V_CMP_EQ_U32_e64 %4.sub0, %4.sub1, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5
+...
+
+---
+name: mfma_waw_boundary_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; GFX90A-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX90A-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-AH-NEXT: {{ $}}
+ ; GFX90A-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX942-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-AH-NEXT: {{ $}}
+ ; GFX942-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX950-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-AH-NEXT: {{ $}}
+ ; GFX950-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub1, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %6, %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %7:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %7, %2, %3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %8:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %8, %2, %3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %9:vgpr_32 = V_ADD_U32_e32 %1.sub1, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %9, %2, %3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %10:vgpr_32 = V_ADD_U32_e32 %1.sub1, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %10, %2, %3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %11:vgpr_32 = V_ADD_U32_e32 %1.sub2, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %11, %2, %3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %12:vgpr_32 = V_ADD_U32_e32 %0.sub0, %1.sub0, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %12, %2, %3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %13:vgpr_32 = V_ADD_U32_e32 %0.sub1, %1.sub1, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %13, %2, %3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %14:vgpr_32 = V_ADD_U32_e32 %0.sub2, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %14, %2, %3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
new file mode 100644
index 0000000000000..9c948123947b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
@@ -0,0 +1,145 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
+
+--- |
+ target triple = "amdgcn-amd-amdhsa"
+
+ define amdgpu_kernel void @test_anti_hints() #0 {
+ bb.0:
+ ret void
+ }
+
+ define amdgpu_kernel void @test_anti_hints_multi_mfma() #0 {
+ bb.0:
+ ret void
+ }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+
+...
+---
+name: test_anti_hints
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; CHECK-LABEL: name: test_anti_hints
+ ; CHECK: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr16, killed renamable $vgpr50_vgpr51_vgpr52_vgpr53, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_anti_hints
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr16, killed renamable $vgpr34_vgpr35_vgpr36_vgpr37, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %4, %7, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+---
+name: test_anti_hints_multi_mfma
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+
+ ; CHECK-LABEL: name: test_anti_hints_multi_mfma
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_anti_hints_multi_mfma
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..487e260b31d83 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -897,26 +897,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; GCN-NEXT: v_mov_b32_e32 v35, s23
; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: v_mov_b32_e32 v16, s16
-; GCN-NEXT: v_mov_b32_e32 v17, s17
-; GCN-NEXT: v_mov_b32_e32 v18, s18
-; GCN-NEXT: v_mov_b32_e32 v19, s19
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
+; GCN-NEXT: s_nop 0
+; GCN-NEXT: v_mov_b32_e32 v32, s16
+; GCN-NEXT: v_mov_b32_e32 v33, s17
+; GCN-NEXT: v_mov_b32_e32 v34, s18
+; GCN-NEXT: v_mov_b32_e32 v35, s19
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s12
-; GCN-NEXT: v_mov_b32_e32 v17, s13
-; GCN-NEXT: v_mov_b32_e32 v18, s14
-; GCN-NEXT: v_mov_b32_e32 v19, s15
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s12
+; GCN-NEXT: v_mov_b32_e32 v33, s13
+; GCN-NEXT: v_mov_b32_e32 v34, s14
+; GCN-NEXT: v_mov_b32_e32 v35, s15
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:16 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s8
-; GCN-NEXT: v_mov_b32_e32 v17, s9
-; GCN-NEXT: v_mov_b32_e32 v18, s10
-; GCN-NEXT: v_mov_b32_e32 v19, s11
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s8
+; GCN-NEXT: v_mov_b32_e32 v33, s9
+; GCN-NEXT: v_mov_b32_e32 v34, s10
+; GCN-NEXT: v_mov_b32_e32 v35, s11
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -1166,26 +1166,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; GCN-NEXT: v_mov_b32_e32 v35, s23
; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: v_mov_b32_e32 v16, s16
-; GCN-NEXT: v_mov_b32_e32 v17, s17
-; GCN-NEXT: v_mov_b32_e32 v18, s18
-; GCN-NEXT: v_mov_b32_e32 v19, s19
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
+; GCN-NEXT: s_nop 0
+; GCN-NEXT: v_mov_b32_e32 v32, s16
+; GCN-NEXT: v_mov_b32_e32 v33, s17
+; GCN-NEXT: v_mov_b32_e32 v34, s18
+; GCN-NEXT: v_mov_b32_e32 v35, s19
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s12
-; GCN-NEXT: v_mov_b32_e32 v17, s13
-; GCN-NEXT: v_mov_b32_e32 v18, s14
-; GCN-NEXT: v_mov_b32_e32 v19, s15
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s12
+; GCN-NEXT: v_mov_b32_e32 v33, s13
+; GCN-NEXT: v_mov_b32_e32 v34, s14
+; GCN-NEXT: v_mov_b32_e32 v35, s15
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:16 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s8
-; GCN-NEXT: v_mov_b32_e32 v17, s9
-; GCN-NEXT: v_mov_b32_e32 v18, s10
-; GCN-NEXT: v_mov_b32_e32 v19, s11
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s8
+; GCN-NEXT: v_mov_b32_e32 v33, s9
+; GCN-NEXT: v_mov_b32_e32 v34, s10
+; GCN-NEXT: v_mov_b32_e32 v35, s11
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..aba29a9a1ec1a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -8244,42 +8244,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; SDAG-NEXT: s_nop 14
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; SDAG-NEXT: s_nop 12
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v20, s14
+; SDAG-NEXT: v_mov_b32_e32 v21, s15
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v22, s10
+; SDAG-NEXT: v_mov_b32_e32 v23, s11
+; SDAG-NEXT: v_mov_b32_e32 v18, s12
+; SDAG-NEXT: v_mov_b32_e32 v19, s13
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v20, s8
+; SDAG-NEXT: v_mov_b32_e32 v21, s9
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8361,42 +8362,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; AGPR-SDAG-NEXT: s_nop 14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; AGPR-SDAG-NEXT: s_nop 12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s9
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..d14fa633a6da1 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -933,10 +933,9 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[2:5], v6, v7, v[0:3]
; GREEDY90A-GISEL-NEXT: s_nop 1
; GREEDY90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v6, v7, v[0:3]
-; GREEDY90A-GISEL-NEXT: s_nop 1
-; GREEDY90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GREEDY90A-GISEL-NEXT: s_nop 1
-; GREEDY90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GREEDY90A-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GREEDY90A-GISEL-NEXT: s_nop 3
+; GREEDY90A-GISEL-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GREEDY90A-GISEL-NEXT: s_endpgm
;
; FAST90A-LABEL: test_mfma_f32_4x4x1f32:
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 8e4832f7f9e6d..db00388565b00 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -376,65 +376,65 @@ define amdgpu_kernel void @illegal_mfma_after_rewrite() #1 {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: s_mov_b32 s5, s4
-; CHECK-NEXT: v_mov_b64_e32 v[22:23], s[4:5]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[4:5]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def s[0:3]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; def v[16:19]
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_mov_b32_e32 v10, 0x7fc00000
; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; CHECK-NEXT: s_mov_b32 s0, 0x3c003c00
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[22:23], v[22:23], v[0:3]
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[0:1]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[24:25], v[24:25], v[0:3]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[0:1]
+; CHECK-NEXT: v_mov_b32_e32 v11, v10
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[24:25], v[24:25], v[4:7]
+; CHECK-NEXT: v_mov_b32_e32 v13, v10
; CHECK-NEXT: s_mov_b32 s0, 0x7e007e00
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[22:23], v[22:23], v[4:7]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[0:1]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[24:25], v[0:3]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[6:9]
-; CHECK-NEXT: s_nop 3
-; CHECK-NEXT: v_cvt_f16_f32_e32 v20, v4
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[12:15], v[22:23], v[26:27], v[0:3]
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: v_mov_b32_e32 v8, 0x7fc00000
-; CHECK-NEXT: v_mov_b32_e32 v9, v8
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
-; CHECK-NEXT: v_mov_b32_e32 v11, v8
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[26:27], v[0:3]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], s[0:1]
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v[18:21]
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_nop 1
+; CHECK-NEXT: v_cvt_f16_f32_e32 v22, v4
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[6:9]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[24:25], v[24:25], v[10:13]
+; CHECK-NEXT: s_nop 5
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[14:17], v[24:25], v[28:29], v[0:3]
+; CHECK-NEXT: s_nop 2
; CHECK-NEXT: v_mov_b64_e32 v[0:1], 0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[22:23], v[22:23], v[8:11]
; CHECK-NEXT: global_store_short v[0:1], v6, off
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[8:11]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[8:11]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[22:23], v[22:23], v[16:19]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[16:19], v[22:23], v[24:25], v[16:19]
-; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[24:25], v[24:25], v[18:21]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[18:21], v[24:25], v[26:27], v[18:21]
+; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_cvt_f16_f32_e32 v10, v6
; CHECK-NEXT: global_store_short v[0:1], v10, off
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[12:15]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[14:17]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[22:23], v[22:23], v[2:5]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[22:23], v[22:23], v[16:19]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[24:25], v[24:25], v[2:5]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[24:25], v[24:25], v[18:21]
; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
; CHECK-NEXT: global_store_short v[0:1], v6, off
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[22:23], v[8:11]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[26:27], v[24:25], v[8:11]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: global_store_short v[0:1], v20, off
+; CHECK-NEXT: global_store_short v[0:1], v22, off
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[26:27], v[22:23], v[2:5]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[28:29], v[24:25], v[2:5]
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
; CHECK-NEXT: global_store_short v[0:1], v6, off
; CHECK-NEXT: s_nop 4
More information about the llvm-branch-commits
mailing list