[llvm] [AMDGPU] Insert MFMA anti-hints in GCNPreRAOptimizations (PR #218075)
Syadus Sefat via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 21:13:44 PDT 2026
https://github.com/mssefat updated https://github.com/llvm/llvm-project/pull/218075
>From c07ea5062c496e42eb8d70103dfdae199a31a7a3 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 19 Aug 2026 13:08:34 -0500
Subject: [PATCH 01/22] [RegAlloc] Add register allocation anti-hints
infrastructure
---
.../llvm/CodeGen/MachineRegisterInfo.h | 59 +++++++++++++++++++
.../include/llvm/CodeGen/TargetRegisterInfo.h | 9 +++
llvm/lib/CodeGen/AllocationOrder.cpp | 36 ++++++++++-
llvm/lib/CodeGen/AllocationOrder.h | 11 ++++
llvm/lib/CodeGen/MachineRegisterInfo.cpp | 22 +++++++
llvm/lib/CodeGen/TargetRegisterInfo.cpp | 40 +++++++++++++
.../unittests/CodeGen/AllocationOrderTest.cpp | 24 ++++++++
7 files changed, 198 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index a26462641de52..890355febbb28 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/IndexedMap.h"
#include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringSet.h"
@@ -42,6 +43,7 @@
namespace llvm {
class PSetIterator;
+class VirtRegMap;
/// Convenient type to represent either a register class or a register bank.
using RegClassOrRegBank =
@@ -119,6 +121,12 @@ class MachineRegisterInfo {
/// pass when deserializing from .mir files.
SmallVector<PendingVirtRegMapEntry, 0> PendingVirtRegMapEntries;
+ /// AntiHintRegs - This vector records register anti-hints for
+ /// virtual registers. For each virtual register, it keeps a vector of virtual
+ /// registers that should NOT be allocated to the same or overlapping physical
+ /// registers.
+ IndexedMap<SmallVector<Register, 4>, VirtReg2IndexFunctor> AntiHintRegs;
+
/// PhysRegUseDefLists - This is an array of the head of the use/def list for
/// physical registers.
std::unique_ptr<MachineOperand *[]> PhysRegUseDefLists;
@@ -906,6 +914,57 @@ class MachineRegisterInfo {
return RegAllocHints.inBounds(VReg) ? &RegAllocHints[VReg] : nullptr;
}
+ /// Add a register allocation anti-hint for the specified virtual register.
+ /// This tells the allocator to avoid allocating VReg to the same physical
+ /// register as AntiHintVReg (or overlapping ones).
+ void addRegAllocAntiHint(Register VReg, Register AntiHintVReg) {
+ assert(VReg.isVirtual() && AntiHintVReg.isVirtual() &&
+ "Anti-hints and anti-hint targets are only for virtual registers");
+ AntiHintRegs.grow(VReg);
+ SmallVector<Register, 4> &AntiHints = AntiHintRegs[VReg];
+ // Avoid duplicates
+ if (!is_contained(AntiHints, AntiHintVReg))
+ AntiHints.push_back(AntiHintVReg);
+ }
+
+ /// Add multiple anti-hints at once.
+ void addRegAllocationAntiHints(Register VReg,
+ ArrayRef<Register> AntiHintVRegs) {
+ for (Register AntiHint : AntiHintVRegs)
+ addRegAllocAntiHint(VReg, AntiHint);
+ }
+
+ /// Clear all anti-hints for a register.
+ void clearRegAllocationAntiHints(Register VReg) {
+ assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
+ if (AntiHintRegs.inBounds(VReg))
+ AntiHintRegs[VReg].clear();
+ }
+
+ /// Return the vector of anti-hints for VReg.
+ ArrayRef<Register> getRegAllocationAntiHints(Register VReg) const {
+ assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
+ if (!AntiHintRegs.inBounds(VReg))
+ return ArrayRef<Register>();
+ return AntiHintRegs[VReg];
+ }
+
+ /// Check if VReg has AntiHintVReg as an anti-hint.
+ bool hasRegAllocationAntiHint(Register VReg, Register AntiHintVReg) const {
+ assert(VReg.isVirtual() && AntiHintVReg.isVirtual() &&
+ "Anti-hints and anti-hint targets are only for virtual registers");
+ if (!AntiHintRegs.inBounds(VReg))
+ return false;
+ const SmallVector<Register, 4> &AntiHints = AntiHintRegs[VReg];
+ return is_contained(AntiHints, AntiHintVReg);
+ }
+
+ /// Get the set of physical registers to avoid.
+ /// VRM is the current virtual register map showing allocations made so far.
+ void getPhysRegAntiHints(Register VReg,
+ SmallVectorImpl<MCPhysReg> &PhysAntiHints,
+ const VirtRegMap &VRM) const;
+
/// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the
/// specified register as undefined which causes the DBG_VALUE to be
/// deleted during LiveDebugVariables analysis.
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 484234fa055fe..e0c358d25a087 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -852,6 +852,15 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
// Do nothing.
}
+ /// Apply anti-hints to the allocation order.
+ virtual void
+ applyRegAllocationAntiHints(Register VirtReg, ArrayRef<MCPhysReg> &Order,
+ SmallVectorImpl<MCPhysReg> &OrderStorage,
+ SmallVectorImpl<MCPhysReg> &AntiHints,
+ const MachineFunction &MF,
+ const VirtRegMap *VRM = nullptr,
+ const LiveRegMatrix *Matrix = nullptr) const;
+
/// Allow the target to reverse allocation order of local live ranges. This
/// will generally allocate shorter local live ranges first. For targets with
/// many registers, this could reduce regalloc compile time by a large
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index 183dc8af1b91b..1b3d3639fbc2a 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -31,6 +31,7 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
const LiveRegMatrix *Matrix) {
const MachineFunction &MF = VRM.getMachineFunction();
const TargetRegisterInfo *TRI = &VRM.getTargetRegInfo();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
auto Order = RegClassInfo.getOrder(MF.getRegInfo().getRegClass(VirtReg));
SmallVector<MCPhysReg, 16> Hints;
bool HardHints =
@@ -44,8 +45,37 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
dbgs() << '\n';
}
});
- assert(all_of(Hints,
- [&](MCPhysReg Hint) { return is_contained(Order, Hint); }) &&
+
+ // Get anti-hints
+ SmallVector<MCPhysReg, 16> AntiHintedPhysRegs;
+ MRI.getPhysRegAntiHints(VirtReg, AntiHintedPhysRegs, VRM);
+
+ LLVM_DEBUG({
+ if (!AntiHintedPhysRegs.empty()) {
+ dbgs() << "anti-hints:";
+ for (MCPhysReg AntiHint : AntiHintedPhysRegs)
+ dbgs() << ' ' << printReg(AntiHint, TRI);
+ dbgs() << '\n';
+ }
+ });
+
+ // Storage for filtered order (used if anti-hints cause reordering)
+ SmallVector<MCPhysReg, 16> ShuffledOrder;
+
+ if (!AntiHintedPhysRegs.empty()) {
+ TRI->applyRegAllocationAntiHints(VirtReg, Order, ShuffledOrder,
+ AntiHintedPhysRegs, MF, &VRM, Matrix);
+ }
+ // Use ShuffledOrder as the order if it was populated by anti-hints
+ // processing
+ ArrayRef<MCPhysReg> FinalOrder =
+ ShuffledOrder.empty() ? Order : ShuffledOrder;
+ // Create allocation order object
+ AllocationOrder AO(std::move(Hints), FinalOrder, HardHints,
+ std::move(ShuffledOrder));
+
+ assert(all_of(AO.Hints,
+ [&](MCPhysReg Hint) { return is_contained(AO.Order, Hint); }) &&
"Target hint is outside allocation order.");
- return AllocationOrder(std::move(Hints), Order, HardHints);
+ return AO;
}
diff --git a/llvm/lib/CodeGen/AllocationOrder.h b/llvm/lib/CodeGen/AllocationOrder.h
index 3dd02c3b14d3a..3b7c7cd4cac10 100644
--- a/llvm/lib/CodeGen/AllocationOrder.h
+++ b/llvm/lib/CodeGen/AllocationOrder.h
@@ -20,6 +20,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/Register.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
namespace llvm {
@@ -29,6 +30,9 @@ class LiveRegMatrix;
class LLVM_LIBRARY_VISIBILITY AllocationOrder {
const SmallVector<MCPhysReg, 16> Hints;
+ // Used as storage if the Order received in the constructor needs to be
+ // altered.
+ SmallVector<MCPhysReg, 16> FilteredOrderStorage;
ArrayRef<MCPhysReg> Order;
// How far into the Order we can iterate. This is 0 if the AllocationOrder is
// constructed with HardHints = true, Order.size() otherwise. While
@@ -92,6 +96,13 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
: Hints(std::move(Hints)), Order(Order),
IterationLimit(HardHints ? 0 : static_cast<int>(Order.size())) {}
+ /// Create an AllocationOrder with pre-computed FilteredOrderStorage.
+ AllocationOrder(SmallVector<MCPhysReg, 16> &&Hints, ArrayRef<MCPhysReg> Order,
+ bool HardHints, SmallVector<MCPhysReg, 16> &&Storage)
+ : Hints(std::move(Hints)), FilteredOrderStorage(std::move(Storage)),
+ Order(FilteredOrderStorage.empty() ? Order : FilteredOrderStorage),
+ IterationLimit(HardHints ? 0 : static_cast<int>(this->Order.size())) {}
+
Iterator begin() const {
return Iterator(*this, -(static_cast<int>(Hints.size())));
}
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index dbba413548018..3918c169b978c 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -11,15 +11,18 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/DebugLoc.h"
@@ -714,3 +717,22 @@ void MachineRegisterInfo::updateDbgUsersToReg(
}
}
}
+
+void MachineRegisterInfo::getPhysRegAntiHints(
+ Register VReg, SmallVectorImpl<MCPhysReg> &PhysAntiHints,
+ const VirtRegMap &VRM) const {
+ assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
+ if (!AntiHintRegs.inBounds(VReg))
+ return;
+
+ const SmallVector<Register, 4> &AntiHints = AntiHintRegs[VReg];
+
+ for (Register AntiHintVReg : AntiHints) {
+ // Check if the anti-hinted register has been allocated
+ if (VRM.hasPhys(AntiHintVReg)) {
+ MCPhysReg PhysReg = VRM.getPhys(AntiHintVReg);
+ if (!is_contained(PhysAntiHints, PhysReg))
+ PhysAntiHints.push_back(PhysReg);
+ }
+ }
+}
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index 3b29b0863dd6c..fd696297e35d9 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -443,6 +443,46 @@ bool TargetRegisterInfo::getRegAllocationHints(
return false;
}
+void TargetRegisterInfo::applyRegAllocationAntiHints(
+ Register VirtReg, ArrayRef<MCPhysReg> &Order,
+ SmallVectorImpl<MCPhysReg> &OrderStorage,
+ SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
+ const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
+
+ if (AntiHints.empty() || !VRM)
+ return;
+
+ auto isAntiHinted = [&](MCPhysReg Reg) {
+ return llvm::any_of(AntiHints, [&](MCPhysReg AntiHint) {
+ return regsOverlap(Reg, AntiHint);
+ });
+ };
+
+ // Copy order into storage
+ OrderStorage.clear();
+ OrderStorage.assign(Order.begin(), Order.end());
+
+ // Partition non-anti-hinted register go first
+ auto PartionPoint =
+ std::stable_partition(OrderStorage.begin(), OrderStorage.end(),
+ [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
+
+ Order = OrderStorage;
+
+ // print the details
+ LLVM_DEBUG({
+ size_t NonAntiHintedCount =
+ std::distance(OrderStorage.begin(), PartionPoint);
+ size_t AntiHintedCount = std::distance(PartionPoint, OrderStorage.end());
+ dbgs() << "Added " << NonAntiHintedCount
+ << " non-anti-hinted registers first\n"
+ << "Added " << AntiHintedCount
+ << " anti-hinted registers at the end\n";
+ });
+
+ return;
+}
+
bool TargetRegisterInfo::isCalleeSavedPhysReg(
MCRegister PhysReg, const MachineFunction &MF) const {
if (!PhysReg)
diff --git a/llvm/unittests/CodeGen/AllocationOrderTest.cpp b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
index d4da8e28ae7f7..ec02579f55c58 100644
--- a/llvm/unittests/CodeGen/AllocationOrderTest.cpp
+++ b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
@@ -116,3 +116,27 @@ TEST(AllocationOrderTest, IsHintTest) {
EXPECT_FALSE(I.isHint());
EXPECT_EQ(V, 5U);
}
+
+TEST(AllocationOrderTest, StorageOverridesOrder) {
+ SmallVector<MCPhysReg, 16> Hints = {1, 2};
+ SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
+ SmallVector<MCPhysReg, 16> Storage = {5, 3, 4};
+ AllocationOrder O(std::move(Hints), Order, false, std::move(Storage));
+ EXPECT_EQ((std::vector<MCPhysReg>{1, 2, 5, 3, 4}), loadOrder(O));
+}
+
+TEST(AllocationOrderTest, EmptyStorageFallsBackToOrder) {
+ SmallVector<MCPhysReg, 16> Hints = {1, 2};
+ SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
+ SmallVector<MCPhysReg, 16> Storage;
+ AllocationOrder O(std::move(Hints), Order, false, std::move(Storage));
+ EXPECT_EQ((std::vector<MCPhysReg>{1, 2, 3, 4, 5}), loadOrder(O));
+}
+
+TEST(AllocationOrderTest, StorageHardHints) {
+ SmallVector<MCPhysReg, 16> Hints = {1, 2};
+ SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
+ SmallVector<MCPhysReg, 16> Storage = {5, 3, 4};
+ AllocationOrder O(std::move(Hints), Order, true, std::move(Storage));
+ EXPECT_EQ((std::vector<MCPhysReg>{1, 2}), loadOrder(O));
+}
>From 244fb01f8407f2138d07509ee26c2d3097bce79d Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 19 Aug 2026 14:18:37 -0500
Subject: [PATCH 02/22] [MIR] Serialize register allocation anti-hints
---
.../include/llvm/CodeGen/MIRParser/MIParser.h | 1 +
llvm/include/llvm/CodeGen/MIRYamlMapping.h | 4 +
llvm/lib/CodeGen/MIRParser/MIRParser.cpp | 18 +++
llvm/lib/CodeGen/MIRPrinter.cpp | 12 ++
...r-allocation-antihints-mir-print-parse.mir | 121 ++++++++++++++++++
5 files changed, 156 insertions(+)
create mode 100644 llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
index 622862279cfdd..d163451a8a588 100644
--- a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
+++ b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h
@@ -47,6 +47,7 @@ struct VRegInfo {
} D;
Register VReg;
Register PreferredReg;
+ SmallVector<Register, 4> AntiHints;
uint8_t Flags = 0;
};
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 76f5e38a045ff..05f6be547f367 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -203,6 +203,7 @@ struct VirtualRegisterDefinition {
StringValue Class;
StringValue PreferredRegister;
std::vector<FlowStringValue> RegisterFlags;
+ std::vector<FlowStringValue> AntiHints;
// VirtRegMap state.
// SplitFrom: id-form virtual register only (e.g. '%0'); physregs and named
// vregs are rejected by the parser.
@@ -227,6 +228,9 @@ template <> struct MappingTraits<VirtualRegisterDefinition> {
StringValue()); // Don't print out when it's empty.
YamlIO.mapOptional("flags", Reg.RegisterFlags,
std::vector<FlowStringValue>());
+ if (!YamlIO.outputting() || !Reg.AntiHints.empty())
+ YamlIO.mapOptional("anti-hints", Reg.AntiHints,
+ std::vector<FlowStringValue>());
// MIRPrinter sets WriteDefaultValues=true unless -simplify-mir is passed,
// so a plain mapOptional with an empty default would still emit the keys
// and change every existing test's output.
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 6f1e7594f34da..8783078485877 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -772,6 +772,22 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS,
FlagStringValue.Value + "'");
Info.Flags |= FlagValue;
}
+ if (!VReg.AntiHints.empty() && Info.Kind != VRegInfo::NORMAL)
+ return error(VReg.AntiHints.front().SourceRange.Start,
+ Twine("anti-hints can only be set for normal vregs"));
+
+ for (const auto &AntiHintValue : VReg.AntiHints) {
+ Register AntiHintReg;
+ if (parseRegisterReference(PFS, AntiHintReg, AntiHintValue.Value, Error))
+ return error(Error, AntiHintValue.SourceRange);
+
+ if (!AntiHintReg.isVirtual())
+ return error(AntiHintValue.SourceRange.Start,
+ Twine("anti-hint '") + AntiHintValue.Value +
+ "' must be a virtual register");
+
+ Info.AntiHints.push_back(AntiHintReg);
+ }
RegInfo.noteNewVirtualRegister(Info.VReg);
}
@@ -878,6 +894,8 @@ bool MIRParserImpl::setupRegisterInfo(const PerFunctionMIParsingState &PFS,
MRI.setRegClass(Reg, Info.D.RC);
if (Info.PreferredReg != 0)
MRI.setSimpleHint(Reg, Info.PreferredReg);
+ if (!Info.AntiHints.empty())
+ MRI.addRegAllocationAntiHints(Reg, Info.AntiHints);
break;
case VRegInfo::GENERIC:
break;
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 8acd6f14ebc2e..d4dff3be9041a 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -323,6 +323,18 @@ static void convertMRI(yaml::MachineFunction &YamlMF, const MachineFunction &MF,
if (PreferredReg)
printRegMIR(PreferredReg, VReg.PreferredRegister, TRI);
printRegFlags(Reg, VReg.RegisterFlags, MF, TRI);
+
+ // Print the anti-hints.
+ const auto &AntiHints = RegInfo.getRegAllocationAntiHints(Reg);
+ if (!AntiHints.empty()) {
+ std::vector<yaml::FlowStringValue> AntiHintStrings;
+ for (Register AntiHint : AntiHints) {
+ yaml::FlowStringValue AntiHintStr;
+ printRegMIR(AntiHint, AntiHintStr, TRI);
+ AntiHintStrings.push_back(std::move(AntiHintStr));
+ }
+ VReg.AntiHints = std::move(AntiHintStrings);
+ }
if (VRM) {
Register Orig = VRM->getPreSplitReg(Reg);
if (Orig && Orig != Reg) {
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir b/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
new file mode 100644
index 0000000000000..574fe61c416de
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
@@ -0,0 +1,121 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=none -o - %s | FileCheck %s
+
+---
+name: single_anti_hint
+tracksRegLiveness: true
+# CHECK-LABEL: name: single_anti_hint
+# CHECK: registers:
+# CHECK: - { id: 3, class: vreg_128_align2, preferred-register: '', flags: [ ],
+# CHECK-NEXT: anti-hints: [ '%1' ] }
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vreg_512_align2 }
+ - { id: 2, class: vreg_512_align2 }
+ - { id: 3, class: vreg_128_align2, anti-hints: [ '%1' ] }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %1.sub0_sub1, %1.sub2_sub3, %1, 0, 0, 0, implicit $mode, implicit $exec
+ %3:vreg_128_align2 = DS_READ_B128_gfx9 %0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %2, implicit %3
+...
+
+---
+name: multiple_anti_hints
+tracksRegLiveness: true
+# CHECK-LABEL: name: multiple_anti_hints
+# CHECK: registers:
+# CHECK: - { id: 3, class: vgpr_32, preferred-register: '', flags: [ ], anti-hints: [
+# CHECK-NEXT: '%1',
+# CHECK-NEXT: '%2' ] }
+# CHECK-NEXT: - { id: 4, class: vreg_128_align2, preferred-register: '', flags: [ ],
+# CHECK-NEXT: anti-hints: [ '%2' ] }
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vreg_512_align2 }
+ - { id: 2, class: vreg_512_align2 }
+ - { id: 3, class: vgpr_32, anti-hints: [ '%1', '%2' ] }
+ - { id: 4, class: vreg_128_align2, anti-hints: [ '%2' ] }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %1.sub0_sub1, %1.sub2_sub3, %1, 0, 0, 0, implicit $mode, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 4096, %0, implicit $exec
+ %4:vreg_128_align2 = DS_READ_B128_gfx9 %3, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %2, implicit %4
+...
+
+---
+name: duplicate_anti_hints
+tracksRegLiveness: true
+# CHECK-LABEL: name: duplicate_anti_hints
+# CHECK: registers:
+# CHECK: - { id: 2, class: vgpr_32, preferred-register: '', flags: [ ], anti-hints: [
+# CHECK-NEXT: '%1' ] }
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vreg_512_align2 }
+ - { id: 2, class: vgpr_32, anti-hints: [ '%1', '%1' ] }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = V_ADD_U32_e32 4096, %0, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2
+...
+
+---
+name: anti_hint_with_preferred_register
+tracksRegLiveness: true
+# CHECK-LABEL: name: anti_hint_with_preferred_register
+# CHECK: registers:
+# CHECK: - { id: 2, class: vgpr_32, preferred-register: '$vgpr8', flags: [ ],
+# CHECK-NEXT: anti-hints: [ '%1' ] }
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vreg_512_align2 }
+ - { id: 2, class: vgpr_32, preferred-register: '$vgpr8', anti-hints: [ '%1' ] }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = V_ADD_U32_e32 4096, %0, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2
+...
+
+---
+name: empty_anti_hints
+tracksRegLiveness: true
+# CHECK-LABEL: name: empty_anti_hints
+# CHECK: registers:
+# CHECK: - { id: 1, class: vgpr_32, preferred-register: '', flags: [ ] }
+# CHECK-NEXT: liveins:
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vgpr_32, anti-hints: [ ] }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = V_ADD_U32_e32 4096, %0, implicit $exec
+ S_ENDPGM 0, implicit %1
+...
+
+---
+name: no_anti_hints
+tracksRegLiveness: true
+# CHECK-LABEL: name: no_anti_hints
+# CHECK: registers:
+# CHECK-NEXT: - { id: 0, class: vgpr_32, preferred-register: '', flags: [ ] }
+# CHECK-NEXT: - { id: 1, class: vgpr_32, preferred-register: '', flags: [ ] }
+# CHECK-NEXT: liveins:
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: vgpr_32 }
+body: |
+ bb.0:
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = V_ADD_U32_e32 4096, %0, implicit $exec
+ S_ENDPGM 0, implicit %1
+...
>From 04a57042c2034b195a86614b938bd3126c13f578 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 19 Aug 2026 20:02:07 -0500
Subject: [PATCH 03/22] [AMDGPU] Apply occupancy-aware register allocation
anti-hints
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 131 +++++-
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 16 +
.../AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 373 ++++++++++++++++++
3 files changed, 519 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 971dedd0878f1..97fca7ca20a26 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -11,18 +11,23 @@
//
//===----------------------------------------------------------------------===//
+#include "SIRegisterInfo.h"
#include "AMDGPU.h"
#include "AMDGPURegisterBankInfo.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUInstPrinter.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
-#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+#define DEBUG_TYPE "amdgpu-si-register-info"
using namespace llvm;
@@ -4173,6 +4178,130 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
}
}
+bool SIRegisterInfo::shouldApplyAntiHints(
+ const MachineFunction &MF, unsigned NumAllocatedVGPRs,
+ unsigned &MaxVGPRsForCurrentOccupancy) const {
+
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
+ unsigned TargetOccupancy = MFI->getOccupancy();
+ unsigned CurrentOccupancy =
+ ST.getOccupancyWithNumVGPRs(NumAllocatedVGPRs, DynamicVGPRBlockSize);
+ MaxVGPRsForCurrentOccupancy =
+ ST.getMaxNumVGPRs(CurrentOccupancy, DynamicVGPRBlockSize);
+
+ LLVM_DEBUG(dbgs() << "anti-hints: " << NumAllocatedVGPRs
+ << " VGPRs allocated, target occupancy " << TargetOccupancy
+ << ", current occupancy " << CurrentOccupancy << '\n');
+
+ // If we are already at lowest occupancy, then there is no need to protect
+ // against occupancy regression.
+ if (CurrentOccupancy == 1)
+ return true;
+
+ // Set max VGPRs for target and current occupancy to early bail out if we are
+ // close to the limit.
+ unsigned MaxVGPRsCutOffForTargetOccupancy =
+ (ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100;
+ unsigned MaxVGPRsCutOffForCurrentOccupancy =
+ (MaxVGPRsForCurrentOccupancy * 95) / 100;
+
+ if (NumAllocatedVGPRs >= MaxVGPRsCutOffForTargetOccupancy) {
+ LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the "
+ << MaxVGPRsCutOffForTargetOccupancy
+ << " VGPR cutoff for target occupancy\n");
+ return false;
+ }
+
+ if (NumAllocatedVGPRs >= MaxVGPRsCutOffForCurrentOccupancy) {
+ LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the "
+ << MaxVGPRsCutOffForCurrentOccupancy
+ << " VGPR cutoff for current occupancy\n");
+ return false;
+ }
+
+ return true;
+}
+
+void SIRegisterInfo::applyRegAllocationAntiHints(
+ Register VirtReg, ArrayRef<MCPhysReg> &Order,
+ SmallVectorImpl<MCPhysReg> &OrderStorage,
+ SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
+ const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
+
+ // Early exit to default order if we have no anti-hints or no VRM.
+ if (AntiHints.empty() || !VRM)
+ return;
+
+ // Get total number of allocated VGPRs to determine the current occupancy.
+ unsigned NumVGPRs = 0;
+ unsigned NumAGPRs = 0;
+ if (Matrix) {
+ for (MCPhysReg Reg : AMDGPU::VGPR_32RegClass)
+ if (Matrix->isPhysRegUsed(Reg))
+ NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
+ for (MCPhysReg Reg : AMDGPU::AGPR_32RegClass)
+ if (Matrix->isPhysRegUsed(Reg))
+ NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
+ }
+ unsigned NumAllocatedVGPRs =
+ AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs);
+
+ // Early exit if we should not apply anti-hints.
+ unsigned MaxVGPRsForCurrentOccupancy = 0;
+ if (!shouldApplyAntiHints(MF, NumAllocatedVGPRs, MaxVGPRsForCurrentOccupancy))
+ return;
+
+ // Returns true if Reg fits within the current occupancy VGPR budget.
+ auto IsWithinBudget = [&](MCPhysReg Reg) -> bool {
+ const TargetRegisterClass *RC = getPhysRegBaseClass(Reg);
+
+ // No VGPR or AGPR usage.
+ if (!RC ||
+ (!isVGPRClass(RC) && !isAGPRClass(RC) && !isVectorSuperClass(RC)))
+ return true;
+
+ unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
+ unsigned Highest = getHWRegIndex(Reg) + NumRegs - 1;
+ return Highest < MaxVGPRsForCurrentOccupancy;
+ };
+
+ // Copy order.
+ OrderStorage.clear();
+ OrderStorage.assign(Order.begin(), Order.end());
+
+ // Helper to check if a register overlaps with any anti-hint.
+ auto isAntiHinted = [&](MCPhysReg Reg) {
+ return llvm::any_of(AntiHints, [&](MCPhysReg AntiHint) {
+ return regsOverlap(Reg, AntiHint);
+ });
+ };
+
+ // Find the cutoff point for the current occupancy VGPR budget.
+ auto *BeyondBudgetStart = llvm::find_if(
+ OrderStorage, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
+
+ // Only shuffle within the current occupancy VGPR budget.
+ auto *PartitionPoint =
+ std::stable_partition(OrderStorage.begin(), BeyondBudgetStart,
+ [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
+
+ Order = OrderStorage;
+ LLVM_DEBUG({
+ size_t NonAntiHintedCount =
+ std::distance(OrderStorage.begin(), PartitionPoint);
+ size_t AntiHintedCount = std::distance(PartitionPoint, BeyondBudgetStart);
+ size_t BeyondBudgetCount =
+ std::distance(BeyondBudgetStart, OrderStorage.end());
+ dbgs() << "Added " << NonAntiHintedCount
+ << " non-anti-hinted registers first\n"
+ << "Added " << AntiHintedCount
+ << " anti-hinted registers at the end\n"
+ << "Beyond current occupancy budget, left: " << BeyondBudgetCount
+ << '\n';
+ });
+}
+
MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
// Not a callee saved register.
return AMDGPU::SGPR30_SGPR31;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index e464c9334ffea..61fbf90d84488 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -15,6 +15,11 @@
#define LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H
#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/MC/MCRegister.h"
#define GET_REGINFO_HEADER
#include "AMDGPUGenRegisterInfo.inc"
@@ -364,6 +369,17 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
const MachineFunction &MF, const VirtRegMap *VRM,
const LiveRegMatrix *Matrix) const override;
+ bool shouldApplyAntiHints(const MachineFunction &MF,
+ unsigned NumAllocatedVGPRs,
+ unsigned &MaxVGPRsForCurrentOccupancy) const;
+
+ void applyRegAllocationAntiHints(
+ Register VirtReg, ArrayRef<MCPhysReg> &Order,
+ SmallVectorImpl<MCPhysReg> &OrderStorage,
+ SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
+ const VirtRegMap *VRM = nullptr,
+ const LiveRegMatrix *Matrix = nullptr) const override;
+
const int *getRegUnitPressureSets(MCRegUnit RegUnit) const override;
MCRegister getReturnAddressReg(const MachineFunction &MF) const;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
new file mode 100644
index 0000000000000..5d698f533acf2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
@@ -0,0 +1,373 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,machineverifier,virtregrewriter,post-RA-hazard-rec %s -o - | FileCheck %s
+
+--- |
+ define amdgpu_kernel void @anti_hints_present() #0 { ret void }
+ define amdgpu_kernel void @anti_hints_absent() #0 { ret void }
+ define amdgpu_kernel void @anti_hints_suppressed_by_target_budget() #0 { ret void }
+ define amdgpu_kernel void @anti_hints_suppressed_by_current_budget() #0 { ret void }
+ attributes #0 = { nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: anti_hints_present
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 8
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sgpr_128 }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: vgpr_32 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: vreg_512_align2 }
+ - { id: 8, class: vreg_512_align2 }
+ - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] }
+ - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 13, class: vreg_512_align2 }
+ - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] }
+ - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 17, class: vreg_512_align2 }
+ - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] }
+ - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+ ; CHECK-LABEL: name: anti_hints_present
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
+
+---
+name: anti_hints_absent
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 8
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sgpr_128 }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: vgpr_32 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: vreg_512_align2 }
+ - { id: 8, class: vreg_512_align2 }
+ - { id: 9, class: vgpr_32 }
+ - { id: 10, class: vgpr_32 }
+ - { id: 11, class: vreg_128_align2 }
+ - { id: 12, class: vgpr_32 }
+ - { id: 13, class: vreg_512_align2 }
+ - { id: 14, class: vgpr_32 }
+ - { id: 15, class: vreg_128_align2 }
+ - { id: 16, class: vgpr_32 }
+ - { id: 17, class: vreg_512_align2 }
+ - { id: 18, class: vreg_128_align2 }
+ - { id: 19, class: vreg_128_align2 }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+ ; CHECK-LABEL: name: anti_hints_absent
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: S_NOP 4
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: S_NOP 5
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_NOP 6
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
+
+---
+name: anti_hints_suppressed_by_target_budget
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 8
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sgpr_128 }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: vgpr_32 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: vreg_512_align2 }
+ - { id: 8, class: vreg_512_align2 }
+ - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] }
+ - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 13, class: vreg_512_align2 }
+ - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] }
+ - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 17, class: vreg_512_align2 }
+ - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] }
+ - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] }
+ - { id: 20, class: vreg_128_align2 }
+ - { id: 21, class: vreg_128_align2 }
+ - { id: 22, class: vreg_128_align2 }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+ - { reg: '$vgpr52_vgpr53_vgpr54_vgpr55' }
+ - { reg: '$vgpr56_vgpr57_vgpr58_vgpr59' }
+ - { reg: '$vgpr60_vgpr61_vgpr62_vgpr63' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; CHECK-LABEL: name: anti_hints_suppressed_by_target_budget
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: S_NOP 4
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: S_NOP 5
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_NOP 6
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ %20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55
+ %21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59
+ %22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit %20, implicit %21, implicit %22
+...
+
+---
+name: anti_hints_suppressed_by_current_budget
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 4
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sgpr_128 }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: vgpr_32 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: vreg_512_align2 }
+ - { id: 8, class: vreg_512_align2 }
+ - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] }
+ - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 13, class: vreg_512_align2 }
+ - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] }
+ - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 17, class: vreg_512_align2 }
+ - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] }
+ - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] }
+ - { id: 20, class: vreg_128_align2 }
+ - { id: 21, class: vreg_128_align2 }
+ - { id: 22, class: vreg_128_align2 }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+ - { reg: '$vgpr52_vgpr53_vgpr54_vgpr55' }
+ - { reg: '$vgpr56_vgpr57_vgpr58_vgpr59' }
+ - { reg: '$vgpr60_vgpr61_vgpr62_vgpr63' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; CHECK-LABEL: name: anti_hints_suppressed_by_current_budget
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: S_NOP 4
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: S_NOP 5
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_NOP 6
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ %20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55
+ %21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59
+ %22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit %20, implicit %21, implicit %22
+...
>From dea57bab26d05d5fdb841530e90c7bfa6f5b49fc Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 19 Aug 2026 23:51:22 -0500
Subject: [PATCH 04/22] [AMDGPU] Add MFMA anti-hints in GCNPreRAOptimizations
---
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp | 510 ++++++++
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h | 124 ++
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 19 +
.../CodeGen/AMDGPU/anti-hints-mfma-war.mir | 559 +++++++++
.../CodeGen/AMDGPU/anti-hints-mfma-waw.mir | 1060 +++++++++++++++++
.../llvm.amdgcn.mfma.anti-hints.gfx942.mir | 145 +++
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 64 +-
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 118 +-
.../AMDGPU/mfma-no-register-aliasing.ll | 7 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 60 +-
11 files changed, 2543 insertions(+), 124 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
create mode 100644 llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..a881750cf4f38 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -132,6 +132,7 @@ add_llvm_target(AMDGPUCodeGen
GCNIterativeScheduler.cpp
GCNMinRegStrategy.cpp
GCNNSAReassign.cpp
+ GCNPreRAAntiHints.cpp
GCNPreRAOptimizations.cpp
GCNPreRALongBranchReg.cpp
GCNRegPressure.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
new file mode 100644
index 0000000000000..231b1adaea871
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
@@ -0,0 +1,510 @@
+//===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Insert register allocation anti-hints.
+///
+//===----------------------------------------------------------------------===//
+
+#include "GCNPreRAAntiHints.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+
+using namespace llvm;
+using namespace llvm::AMDGPU;
+
+#define DEBUG_TYPE "amdgpu-anti-hints"
+
+namespace HC = llvm::AMDGPU::HazardClass;
+
+static cl::opt<std::string> AntiHintRuleSelection(
+ "amdgpu-anti-hints-rules", cl::Hidden,
+ cl::desc("Comma-separated anti-hints rules (waw, war), or all or none."),
+ cl::init("all"));
+
+namespace {
+
+// Classify the MI into a HazardClassMask.
+HazardClassMask getInstHazardClass(const MachineInstr &MI,
+ const HazardContext &Ctx) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ HazardClassMask Mask = HC::None;
+
+ if (TII.isLDSDMA(MI))
+ Mask = HC::VALU | HC::VMEM | HC::DS;
+ else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
+ Mask = HC::WMMA;
+ else if (TII.isMFMA(MI))
+ Mask = HC::MFMA;
+ else if (SIInstrInfo::isTRANS(MI))
+ Mask = HC::TRANS;
+ else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
+ Mask = HC::VALU;
+ else if (TII.isDS(MI))
+ Mask = HC::DS;
+ else if (TII.isVMEM(MI))
+ Mask = HC::VMEM;
+ else if (TII.isSMRD(MI))
+ Mask = HC::SMEM;
+ else if (TII.isEXP(MI))
+ Mask = HC::EXP;
+ else if (SIInstrInfo::isSALU(MI))
+ Mask = HC::SALU;
+
+ return Mask;
+}
+
+void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
+ const HazardContext &Ctx,
+ SmallVectorImpl<Register> &Out) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ auto Add = [&](const MachineOperand *MO) {
+ if (MO && MO->isReg() && MO->getReg().isVirtual() &&
+ Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO->getReg())))
+ Out.push_back(MO->getReg());
+ };
+ auto Named = [&](AMDGPU::OpName N) { Add(TII.getNamedOperand(MI, N)); };
+ switch (Op) {
+ case HazardOperand::None:
+ break;
+ case HazardOperand::Def:
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isDef())
+ Add(&MO);
+ break;
+ case HazardOperand::Src0:
+ Named(AMDGPU::OpName::src0);
+ break;
+ case HazardOperand::Src1:
+ Named(AMDGPU::OpName::src1);
+ break;
+ case HazardOperand::Src2:
+ Named(AMDGPU::OpName::src2);
+ break;
+ case HazardOperand::Idx:
+ Named(AMDGPU::OpName::idx);
+ break;
+ case HazardOperand::Vaddr:
+ Named(AMDGPU::OpName::vaddr);
+ break;
+ case HazardOperand::AnySrc:
+ Named(AMDGPU::OpName::src0);
+ Named(AMDGPU::OpName::src1);
+ Named(AMDGPU::OpName::src2);
+ break;
+ case HazardOperand::AnyUse:
+ for (const MachineOperand &MO : MI.uses())
+ if (MO.isReg() && MO.isUse())
+ Add(&MO);
+ break;
+ }
+}
+
+enum class MFMAHazardKind { RAW, WAW, WAR };
+
+// MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait
+// states.
+unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
+ HazardClassMask ReaderClass, const HazardContext &Ctx) {
+ const SIInstrInfo &TII = *Ctx.TII;
+ const GCNSubtarget &ST = *Ctx.ST;
+ const int NumPasses = Ctx.SchedModel->computeInstrLatency(&MFMA);
+ const bool IsDGEMM = SIInstrInfo::isDGEMM(MFMA.getOpcode());
+ const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP);
+
+ auto GFX940NPass = [&]() -> unsigned {
+ return TII.isXDL(MFMA)
+ ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
+ : NumPasses + 2;
+ };
+ auto SMFMANPass = [&]() -> unsigned {
+ switch (NumPasses) {
+ case 2:
+ return 5;
+ case 8:
+ return 11;
+ case 16:
+ return 19;
+ default:
+ return 0;
+ }
+ };
+
+ switch (Kind) {
+ case MFMAHazardKind::RAW:
+ if (IsDGEMM) {
+ switch (NumPasses) {
+ case 4:
+ return Mem ? 9 : 6;
+ case 8:
+ case 16:
+ return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
+ default:
+ return 0;
+ }
+ }
+ return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
+
+ case MFMAHazardKind::WAW:
+ if (IsDGEMM) {
+ switch (NumPasses) {
+ case 4:
+ return 6;
+ case 8:
+ case 16:
+ return 11;
+ default:
+ return 0;
+ }
+ }
+ return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
+
+ case MFMAHazardKind::WAR:
+ switch (NumPasses) {
+ case 2:
+ return 1;
+ case 4:
+ return 3;
+ case 8:
+ return 7;
+ case 16:
+ return 15;
+ default:
+ return 15;
+ }
+ }
+ return 0;
+}
+
+unsigned mfmaWawWindow(const MachineInstr &P, const HazardContext &Ctx) {
+ return mfmaWaitStates(P, MFMAHazardKind::WAW, HC::None, Ctx);
+}
+unsigned mfmaWarWindow(const MachineInstr &P, const HazardContext &Ctx) {
+ return mfmaWaitStates(P, MFMAHazardKind::WAR, HC::None, Ctx);
+}
+
+unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
+ HazardClassMask ReaderClass,
+ const HazardContext &Ctx) {
+ return mfmaWaitStates(Producer, MFMAHazardKind::RAW, ReaderClass, Ctx);
+}
+
+bool hasMFMAHazard(const HazardContext &Ctx) {
+ return Ctx.ST->hasGFX90AInsts();
+}
+
+bool ruleSelected(StringRef Name) {
+ StringRef Selection(AntiHintRuleSelection);
+ if (Selection.equals_insensitive("all"))
+ return true;
+ if (Selection.equals_insensitive("none"))
+ return false;
+ SmallVector<StringRef, 3> Selected;
+ Selection.split(Selected, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+ return llvm::any_of(Selected, [Name](StringRef S) {
+ return S.trim().equals_insensitive(Name);
+ });
+}
+
+bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
+ return hasMFMAHazard(Ctx) && ruleSelected("waw");
+}
+
+bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
+ return hasMFMAHazard(Ctx) && ruleSelected("war");
+}
+
+bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
+ return Ctx.TII->isXDL(MI);
+}
+
+unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
+ const HazardContext &Ctx) {
+ // Explicity given window length overrides the computed one.
+ if (CT.Window.OptWindowLength &&
+ CT.Window.OptWindowLength->getNumOccurrences())
+ return *CT.Window.OptWindowLength;
+ if (CT.Window.Fn)
+ return CT.Window.Fn(MI, Ctx);
+ return CT.Window.WindowLength;
+}
+
+// Build the anti-hints rules.
+class HazardRuleSet {
+ SmallVector<HazardAntiHintRule, 0> Rules;
+
+public:
+ class RuleBuilder {
+
+ HazardRuleSet &S;
+ unsigned Idx;
+ HazardAntiHintRule &rule() const { return S.Rules[Idx]; }
+
+ public:
+ RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {}
+
+ RuleBuilder &producer(ClassMatch M, HazardOperand Op,
+ InstPredicate Predicate = nullptr) {
+ rule().Producer = {M, Op, Predicate};
+ return *this;
+ }
+
+ RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) {
+ rule().AdvanceForRawWindow = AdvanceForRawWindow;
+ return *this;
+ }
+
+ RuleBuilder &consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
+ HazardClassMask CountMask = 0,
+ ConsumerHint Hint = ConsumerHint::OneDirectional,
+ InstPredicate Predicate = nullptr) {
+ rule().Consumers.push_back({{M, Op, Predicate}, Window, CountMask, Hint});
+ return *this;
+ }
+
+ RuleBuilder &enabledIf(RulePredicate Predicate) {
+ rule().Predicate = Predicate;
+ return *this;
+ }
+ };
+
+ RuleBuilder addRule() {
+ Rules.emplace_back();
+ return RuleBuilder(*this, Rules.size() - 1);
+ }
+
+ SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); }
+};
+
+// Here the anti-hints rules are inserted.
+SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
+ using HO = HazardOperand;
+ HazardRuleSet S;
+
+ const WindowSpec MfmaWawWindow{0, nullptr, mfmaWawWindow};
+ const WindowSpec MfmaWarWindow{0, nullptr, mfmaWarWindow};
+ const ClassMatch MfmaConsumers = {HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
+ HC::EXP};
+
+ // MFMA WAW rules
+ S.addRule()
+ .enabledIf(isMFMAWAWRuleEnabled)
+ .producer({HC::MFMA}, HO::Def)
+ .rawCredit(mfmaReaderRawWindow)
+ .consumer(MfmaConsumers, HO::Def, MfmaWawWindow, HC::None,
+ ConsumerHint::OneDirectional);
+
+ // MFMA WAR rules
+ S.addRule()
+ .enabledIf(isMFMAWARRuleEnabled)
+ .producer({HC::MFMA}, HO::Src2, isXDLMFMA)
+ .rawCredit(mfmaReaderRawWindow)
+ .consumer(MfmaConsumers, HO::Def, MfmaWarWindow, HC::None,
+ ConsumerHint::OneDirectional);
+
+ return S.buildRules();
+}
+
+ArrayRef<HazardAntiHintRule> getAntiHintsRules() {
+ static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules();
+ return Rules;
+}
+
+struct AntiHintWindow {
+ SmallVector<Register, 4> Regs;
+ const MachineInstr *Producer = nullptr;
+ unsigned Len = 0;
+ unsigned Elapsed = 0;
+};
+
+using ConsumerTracking = SmallVector<AntiHintWindow, 3>;
+// One per consumer target of a rule.
+using RuleTracking = SmallVector<ConsumerTracking, 3>;
+
+class AntiHintEngine {
+ const HazardContext &Ctx;
+ ArrayRef<HazardAntiHintRule> Rules;
+
+ SmallVector<bool, 8> RuleApplies;
+ bool AnyEnabled = false;
+
+public:
+ AntiHintEngine(const HazardContext &Ctx)
+ : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx);
+ AnyEnabled |= RuleApplies[R];
+ }
+ }
+
+ void run(MachineFunction &MF) {
+ if (!AnyEnabled)
+ return;
+
+ SmallVector<RuleTracking, 8> Tracking(Rules.size());
+ for (unsigned R = 0; R < Rules.size(); ++R)
+ Tracking[R].resize(Rules[R].Consumers.size());
+
+ for (const MachineBasicBlock &MBB : MF) {
+ for (RuleTracking &RT : Tracking)
+ for (ConsumerTracking &Track : RT)
+ Track.clear();
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isMetaInstruction())
+ continue;
+ const HazardClassMask C = getInstHazardClass(MI, Ctx);
+ // Wait states this instruction contributes to an open window.
+ unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI);
+ addAntiHintsAndExpire(MI, C, WaitStates, Tracking);
+ addWindows(MI, C, Tracking);
+ }
+ }
+ }
+
+private:
+ // Check if the right class and predicate matches.
+ bool sideMatches(const HazardSide &Side, const MachineInstr &MI,
+ HazardClassMask C) {
+ return Side.Match.matches(C) &&
+ (!Side.Predicate || Side.Predicate(MI, Ctx));
+ }
+
+ // Producer phase: open a window for each consumer target.
+ void addWindows(const MachineInstr &MI, HazardClassMask C,
+ MutableArrayRef<RuleTracking> Tracking) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ if (!RuleApplies[R] || !sideMatches(Rule.Producer, MI, C))
+ continue;
+
+ SmallVector<Register, 4> Regs;
+ // Collect the producer regs.
+ collectOperandRegs(MI, Rule.Producer.Op, Ctx, Regs);
+ if (Regs.empty())
+ continue;
+ for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
+ ConsumerIdx != E; ++ConsumerIdx) {
+ unsigned Window = resolveWindow(Rule.Consumers[ConsumerIdx], MI, Ctx);
+ if (!Window)
+ continue;
+ Tracking[R][ConsumerIdx].push_back({Regs, &MI, Window, 0});
+ }
+ }
+ }
+
+ void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
+ const HazardAntiHintRule &Rule,
+ const ConsumerTarget &CT, ConsumerTracking &Track) {
+
+ if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(C))
+ return;
+ for (AntiHintWindow &Window : Track) {
+ // Determine if def generated by producer is read by the MI consumer.
+ bool ReadsProducerDef = llvm::any_of(
+ Window.Producer->all_defs(), [&](const MachineOperand &MO) {
+ return MO.getReg().isVirtual() &&
+ MI.readsVirtualRegister(MO.getReg());
+ });
+ // Advance by RAW window from the producer if that is larger than current
+ // Window.Elapsed.
+ if (ReadsProducerDef)
+ Window.Elapsed = std::max(
+ Window.Elapsed, Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx));
+ }
+ }
+
+ // Consumer phase: add anti-hints, then charge and expire open windows.
+ void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C,
+ unsigned WaitStates,
+ MutableArrayRef<RuleTracking> Tracking) {
+ for (unsigned R = 0; R < Rules.size(); ++R) {
+ const HazardAntiHintRule &Rule = Rules[R];
+ if (!RuleApplies[R])
+ continue;
+ for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
+ ConsumerIdx != E; ++ConsumerIdx) {
+ const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx];
+ ConsumerTracking &Track = Tracking[R][ConsumerIdx];
+ if (Track.empty())
+ continue;
+
+ // Before adding the anti-hints, see if advancing by RAW window will
+ // help remove the window.
+ advanceByRawWindow(MI, C, Rule, CT, Track);
+ llvm::erase_if(Track, [](const AntiHintWindow &Window) {
+ return Window.Elapsed >= Window.Len;
+ });
+
+ // Add anti-hints if the consumer matches the instruction.
+ if (sideMatches(CT.Side, MI, C))
+ addAntiHints(CT, MI, Track);
+
+ // This instruction's own wait states count toward the next one.
+ if (!CT.CounterMask || (C & CT.CounterMask))
+ for (AntiHintWindow &Window : Track)
+ Window.Elapsed += WaitStates;
+ }
+ }
+ }
+
+ bool isCopyOf(Register Cand, Register HazardReg) const {
+ const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Cand);
+ return Def && Def->isCopy() && Def->getOperand(1).getReg() == HazardReg;
+ }
+
+ void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI,
+ const ConsumerTracking &Track) {
+ if (Track.empty())
+ return;
+ SmallVector<Register, 4> ConsumerRegs;
+ // Collect the consumer regs.
+ collectOperandRegs(MI, CT.Side.Op, Ctx, ConsumerRegs);
+ if (ConsumerRegs.empty())
+ return;
+ SlotIndex Slot = Ctx.LIS->getInstructionIndex(MI).getRegSlot();
+ auto AntiHint = [&](Register ProducerReg) {
+ if (!Ctx.LIS->hasInterval(ProducerReg))
+ return;
+ const LiveInterval &ProducerLI = Ctx.LIS->getInterval(ProducerReg);
+ if (ProducerLI.liveAt(Slot))
+ return;
+ for (Register ConsumerReg : ConsumerRegs) {
+ if (ConsumerReg == ProducerReg || isCopyOf(ConsumerReg, ProducerReg))
+ continue;
+ // Only allow anti-hints when producer and consumer reg live ranges do
+ // not overlap
+ if (Ctx.LIS->hasInterval(ConsumerReg) &&
+ ProducerLI.overlaps(Ctx.LIS->getInterval(ConsumerReg)))
+ continue;
+ Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
+ if (CT.Hint == ConsumerHint::Symmetric)
+ Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
+ LLVM_DEBUG(
+ dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
+ << (CT.Hint == ConsumerHint::Symmetric ? " <-> " : " <- ")
+ << printReg(ConsumerReg, Ctx.TRI) << " (consumer "
+ << Ctx.TII->getName(MI.getOpcode()) << ")\n");
+ }
+ };
+ for (const AntiHintWindow &Window : Track)
+ for (Register ProducerReg : Window.Regs)
+ AntiHint(ProducerReg);
+ }
+};
+
+} // namespace
+
+void AMDGPU::applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx) {
+ AntiHintEngine(Ctx).run(MF);
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
new file mode 100644
index 0000000000000..e9b0e884d549b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
@@ -0,0 +1,124 @@
+//===- GCNPreRAAntiHints.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
+#define LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+class MachineFunction;
+class MachineInstr;
+class MachineRegisterInfo;
+class LiveIntervals;
+class SIRegisterInfo;
+class SIInstrInfo;
+class GCNSubtarget;
+class TargetSchedModel;
+
+namespace AMDGPU {
+
+using HazardClassMask = uint32_t;
+namespace HazardClass {
+
+// Coherent with CoExec instruction types
+enum : HazardClassMask {
+ None = 0,
+ CTRL = 1u << 0,
+ VALU = 1u << 1,
+ TRANS = 1u << 2,
+ SALU = 1u << 3,
+ DS = 1u << 4,
+ VMEM = 1u << 5,
+ SMEM = 1u << 6,
+ WMMA = 1u << 7,
+ MFMA = 1u << 8,
+ EXP = 1u << 9,
+};
+} // namespace HazardClass
+
+enum class HazardOperand : uint8_t {
+ None,
+ Def,
+ Src0,
+ Src1,
+ Src2,
+ Idx,
+ Vaddr,
+ AnySrc,
+ AnyUse,
+};
+
+enum class ConsumerHint : uint8_t {
+ OneDirectional,
+ Symmetric,
+};
+
+struct HazardContext {
+ const SIInstrInfo *TII;
+ const SIRegisterInfo *TRI;
+ MachineRegisterInfo *MRI;
+ LiveIntervals *LIS;
+ const GCNSubtarget *ST;
+ const TargetSchedModel *SchedModel;
+};
+
+struct ClassMatch {
+ HazardClassMask AnyOf = 0;
+ bool matches(HazardClassMask Mask) const { return !AnyOf || (Mask & AnyOf); }
+};
+
+// To enable or disable the whole rule.
+using RulePredicate = bool (*)(const HazardContext &);
+
+// To gate the rule for producer or consumer.
+using InstPredicate = bool (*)(const MachineInstr &, const HazardContext &);
+
+// This helps advancing a window for the WAW/WAR case where a RAW sits between
+// the producer and the consumer.
+using AdvanceForRawWindowFn = unsigned (*)(const MachineInstr &Producer,
+ HazardClassMask ReaderClass,
+ const HazardContext &Ctx);
+
+struct HazardSide {
+ ClassMatch Match;
+ HazardOperand Op = HazardOperand::None;
+ InstPredicate Predicate = nullptr;
+};
+
+// Length of a hazard window in wait states.
+// Precedence: OptWindowLength --> Fn --> WindowLength.
+struct WindowSpec {
+ unsigned WindowLength = 0;
+ const cl::opt<unsigned> *OptWindowLength = nullptr;
+ unsigned (*Fn)(const MachineInstr &Producer,
+ const HazardContext &Ctx) = nullptr;
+};
+
+struct ConsumerTarget {
+ HazardSide Side;
+ WindowSpec Window;
+ HazardClassMask CounterMask = HazardClass::None;
+ ConsumerHint Hint = ConsumerHint::OneDirectional;
+};
+
+struct HazardAntiHintRule {
+ RulePredicate Predicate = nullptr;
+ HazardSide Producer;
+ SmallVector<ConsumerTarget, 3> Consumers;
+ AdvanceForRawWindowFn AdvanceForRawWindow = nullptr;
+};
+
+void applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx);
+
+} // namespace AMDGPU
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index 50514e819f412..3c9c5fef86779 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -32,17 +32,27 @@
#include "GCNPreRAOptimizations.h"
#include "AMDGPU.h"
+#include "GCNPreRAAntiHints.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/Register.h"
+#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
using namespace llvm;
#define DEBUG_TYPE "amdgpu-pre-ra-optimizations"
+static cl::opt<bool> EnableAntiHintsForMFMARegs(
+ "amdgpu-anti-hints-for-mfma", cl::Hidden,
+ cl::desc("Add register allocation anti-hints for MFMA hazards."),
+ cl::init(true));
+
namespace {
class GCNPreRAOptimizationsImpl {
@@ -51,6 +61,7 @@ class GCNPreRAOptimizationsImpl {
const SIRegisterInfo *TRI;
MachineRegisterInfo *MRI;
LiveIntervals *LIS;
+ TargetSchedModel SchedModel;
bool processReg(Register Reg);
void hintTrue16Copy(const MachineInstr &MI);
@@ -305,6 +316,14 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
+ // Anti-hints only steer register allocation, so they do not count as a
+ // modification of the function.
+ if (EnableAntiHintsForMFMARegs) {
+ SchedModel.init(&ST);
+ AMDGPU::HazardContext HCtx{TII, TRI, MRI, LIS, &ST, &SchedModel};
+ AMDGPU::applyAntiHintRules(MF, HCtx);
+ }
+
bool Changed = false;
for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
new file mode 100644
index 0000000000000..148d2ced6399f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
@@ -0,0 +1,559 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX90A
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX942
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX950
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX90A-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX942-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX950-NOAH
+
+--- |
+ define amdgpu_kernel void @mfma_war_xdl_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_vmem() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_xdl_8pass_ds() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_nonxdl_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_nonxdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_war_multi_mfma() #0 { ret void }
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+---
+name: mfma_war_xdl_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 6
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 2
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_4pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 2
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_128_align2 = IMPLICIT_DEF
+ %7:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ $vgpr40_vgpr41_vgpr42_vgpr43 = COPY %7
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+ - { reg: '$vgpr17' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16, $vgpr17
+ ; AH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; AH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 4096, $vgpr16, implicit $exec
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vgpr_32 = COPY $vgpr17
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 4096, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_vmem
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_vmem
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_xdl_8pass_ds
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_xdl_8pass_ds
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %7
+...
+
+---
+name: mfma_war_nonxdl_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-GFX90A-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX90A: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX90A-NEXT: {{ $}}
+ ; AH-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; AH-GFX942-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX942: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX942-NEXT: {{ $}}
+ ; AH-GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX942-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; AH-GFX950-LABEL: name: mfma_war_nonxdl_8pass
+ ; AH-GFX950: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX950-NEXT: {{ $}}
+ ; AH-GFX950-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX950-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX950-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 6
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_nonxdl_8pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_128_align2 = IMPLICIT_DEF
+ %7:vreg_128_align2 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ $vgpr40_vgpr41_vgpr42_vgpr43 = COPY %7
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit $vgpr40_vgpr41_vgpr42_vgpr43, implicit %8
+...
+
+---
+name: mfma_war_nonxdl_16pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-GFX90A-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX90A: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX90A-NEXT: {{ $}}
+ ; AH-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX90A-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX90A-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; AH-GFX942-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX942: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX942-NEXT: {{ $}}
+ ; AH-GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX942-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX942-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; AH-GFX950-LABEL: name: mfma_war_nonxdl_16pass
+ ; AH-GFX950: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-GFX950-NEXT: {{ $}}
+ ; AH-GFX950-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-GFX950-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-GFX950-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-GFX950-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_nonxdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %7, implicit %8
+...
+
+---
+name: mfma_war_multi_mfma
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+ ; AH-LABEL: name: mfma_war_multi_mfma
+ ; AH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX90A-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 12
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 13
+ ; GFX90A-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 14
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX942-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 4
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 5
+ ; GFX942-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_war_multi_mfma
+ ; GFX950-NOAH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 4
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 5
+ ; GFX950-NOAH-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX950-NOAH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
new file mode 100644
index 0000000000000..8404e34080096
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
@@ -0,0 +1,1060 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX90A-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX942-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX950-AH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX90A-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX942-NOAH
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX950-NOAH
+
+--- |
+ define amdgpu_kernel void @mfma_waw_nonxdl_2pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_2pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_4pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_vmem() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_ds() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_8pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_16pass() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_4pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_xdl_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_nonxdl_16pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_4pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_dgemm_8pass_valu() #0 { ret void }
+ define amdgpu_kernel void @mfma_waw_boundary_4pass_valu() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: mfma_waw_nonxdl_2pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 4
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 3
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_2pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 3
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_4X4X1F32_vgprcd_e64 %0.sub0, %0.sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_2pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_2pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; NOAH-LABEL: name: mfma_waw_xdl_2pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 4
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_4X4X4F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_dgemm_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_dgemm_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr12_vgpr13 = BUFFER_LOAD_DWORDX2_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_4pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 5
+ ; NOAH-NEXT: renamable $vgpr10_vgpr11 = BUFFER_LOAD_DWORDX2_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_64_align2 = IMPLICIT_DEF
+ %5:vreg_64_align2 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_64_align2 = BUFFER_LOAD_DWORDX2_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_4pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_4pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_4pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 9
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_8pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 9
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X4F32_vgprcd_e64 %0.sub0, %0.sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_xdl_8pass_vmem
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_vmem
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_8pass_ds
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_ds
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %7
+...
+
+---
+name: mfma_waw_dgemm_8pass
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_dgemm_8pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr26_vgpr27_vgpr28_vgpr29 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr26_vgpr27_vgpr28_vgpr29
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_8pass
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 10
+ ; NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_256_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_16pass
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 1
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 1
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 1
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35_vgpr36_vgpr37
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_16pass
+tracksRegLiveness: true
+machineFunctionInfo:
+ occupancy: 1
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_16pass
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr82_vgpr83_vgpr84_vgpr85 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr82_vgpr83_vgpr84_vgpr85
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 2
+ ; GFX942-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_16pass
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 3
+ ; GFX950-NOAH-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, implicit killed renamable $vgpr50_vgpr51_vgpr52_vgpr53
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_1024_align2 = IMPLICIT_DEF
+ %7:vreg_1024_align2 = contract V_MFMA_F32_32X32X4F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6, implicit %8
+...
+
+---
+name: mfma_waw_xdl_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr18_vgpr19 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr18_vgpr19, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_4pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr12_vgpr13, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %6, %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4
+...
+
+---
+name: mfma_waw_xdl_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr50_vgpr51, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_xdl_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 11
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %8, %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_nonxdl_16pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; AH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr50_vgpr51 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr50_vgpr51, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 15
+ ; GFX90A-NOAH-NEXT: S_NOP 2
+ ; GFX90A-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 15
+ ; GFX942-NOAH-NEXT: S_NOP 1
+ ; GFX942-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_nonxdl_16pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr0, $vgpr1, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 15
+ ; GFX950-NOAH-NEXT: S_NOP 1
+ ; GFX950-NOAH-NEXT: renamable $vgpr34_vgpr35 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORDX2_OFFEN killed renamable $vgpr34_vgpr35, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:sgpr_128 = IMPLICIT_DEF
+ %6:vreg_512_align2 = IMPLICIT_DEF
+ %7:vreg_512_align2 = contract V_MFMA_F32_32X32X2F32_vgprcd_e64 %0.sub0, %0.sub1, %6, 0, 0, 0, implicit $mode, implicit $exec
+ %8:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ BUFFER_STORE_DWORDX2_OFFEN %8, %4, %5, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+
+---
+name: mfma_waw_dgemm_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-LABEL: name: mfma_waw_dgemm_4pass_valu
+ ; AH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; AH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr12_vgpr13 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr12, $vgpr13, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; NOAH-LABEL: name: mfma_waw_dgemm_4pass_valu
+ ; NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr10_vgpr11 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_NOP 5
+ ; NOAH-NEXT: renamable $vgpr10_vgpr11 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr10, $vgpr11, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $sgpr0_sgpr1
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %3:vreg_64_align2 = contract V_MFMA_F64_4X4X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %2, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ %5:sreg_64_xexec = V_CMP_EQ_U32_e64 %4.sub0, %4.sub1, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5
+...
+
+---
+name: mfma_waw_dgemm_8pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; AH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; AH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr24_vgpr25 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; AH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr24, $vgpr25, implicit $exec
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX942-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 10
+ ; GFX942-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_dgemm_8pass_valu
+ ; GFX950-NOAH: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 10
+ ; GFX950-NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr4_vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 6
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 killed $vgpr16, $vgpr17, implicit $exec
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0_sgpr1
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vreg_256_align2 = contract V_MFMA_F64_16X16X4F64_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %2, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_64_align2 = V_LSHLREV_B64_e64 1, %1.sub0_sub1, implicit $exec
+ %5:sreg_64_xexec = V_CMP_EQ_U32_e64 %4.sub0, %4.sub1, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5
+...
+
+---
+name: mfma_waw_boundary_4pass_valu
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr16
+ ; GFX90A-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX90A-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-AH-NEXT: {{ $}}
+ ; GFX90A-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX90A-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX942-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-AH-NEXT: {{ $}}
+ ; GFX942-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX942-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-AH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX950-AH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-AH-NEXT: {{ $}}
+ ; GFX950-AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-AH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-AH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr17, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX950-AH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX90A-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX90A-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX90A-NOAH-NEXT: {{ $}}
+ ; GFX90A-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NOAH-NEXT: S_NOP 10
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX90A-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX90A-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX942-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX942-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX942-NOAH-NEXT: {{ $}}
+ ; GFX942-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX942-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NOAH-NEXT: S_NOP 6
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX942-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX942-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ ;
+ ; GFX950-NOAH-LABEL: name: mfma_waw_boundary_4pass_valu
+ ; GFX950-NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+ ; GFX950-NOAH-NEXT: {{ $}}
+ ; GFX950-NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX950-NOAH-NEXT: dead renamable $vgpr12_vgpr13_vgpr14_vgpr15 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr8_vgpr9_vgpr10_vgpr11, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NOAH-NEXT: S_NOP 7
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr4, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr5, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr6, $vgpr7, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr0, $vgpr4, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr1, $vgpr5, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 $vgpr2, $vgpr6, implicit $exec
+ ; GFX950-NOAH-NEXT: BUFFER_STORE_DWORD_OFFEN killed renamable $vgpr12, renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ ; GFX950-NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vgpr_32 = COPY $vgpr16
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vreg_128_align2 = IMPLICIT_DEF
+ %5:vreg_128_align2 = contract V_MFMA_F32_16X16X16F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %4, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub1, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %6, %2, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %7:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %7, %2, %3, 4, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %8:vgpr_32 = V_ADD_U32_e32 %1.sub0, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %8, %2, %3, 8, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %9:vgpr_32 = V_ADD_U32_e32 %1.sub1, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %9, %2, %3, 12, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %10:vgpr_32 = V_ADD_U32_e32 %1.sub1, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %10, %2, %3, 16, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %11:vgpr_32 = V_ADD_U32_e32 %1.sub2, %1.sub3, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %11, %2, %3, 20, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %12:vgpr_32 = V_ADD_U32_e32 %0.sub0, %1.sub0, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %12, %2, %3, 24, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %13:vgpr_32 = V_ADD_U32_e32 %0.sub1, %1.sub1, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %13, %2, %3, 28, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ %14:vgpr_32 = V_ADD_U32_e32 %0.sub2, %1.sub2, implicit $exec
+ BUFFER_STORE_DWORD_OFFEN %14, %2, %3, 32, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
new file mode 100644
index 0000000000000..9c948123947b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
@@ -0,0 +1,145 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
+
+--- |
+ target triple = "amdgcn-amd-amdhsa"
+
+ define amdgpu_kernel void @test_anti_hints() #0 {
+ bb.0:
+ ret void
+ }
+
+ define amdgpu_kernel void @test_anti_hints_multi_mfma() #0 {
+ bb.0:
+ ret void
+ }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+
+...
+---
+name: test_anti_hints
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; CHECK-LABEL: name: test_anti_hints
+ ; CHECK: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr50_vgpr51_vgpr52_vgpr53 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr16, killed renamable $vgpr50_vgpr51_vgpr52_vgpr53, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_anti_hints
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr34_vgpr35_vgpr36_vgpr37 = DS_READ_B128_gfx9 renamable $vgpr16, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr16, killed renamable $vgpr34_vgpr35_vgpr36_vgpr37, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %5, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %4, %7, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %6
+...
+---
+name: test_anti_hints_multi_mfma
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
+
+ ; CHECK-LABEL: name: test_anti_hints_multi_mfma
+ ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_anti_hints_multi_mfma
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X16_FP8_FP8_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..487e260b31d83 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -897,26 +897,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; GCN-NEXT: v_mov_b32_e32 v35, s23
; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: v_mov_b32_e32 v16, s16
-; GCN-NEXT: v_mov_b32_e32 v17, s17
-; GCN-NEXT: v_mov_b32_e32 v18, s18
-; GCN-NEXT: v_mov_b32_e32 v19, s19
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
+; GCN-NEXT: s_nop 0
+; GCN-NEXT: v_mov_b32_e32 v32, s16
+; GCN-NEXT: v_mov_b32_e32 v33, s17
+; GCN-NEXT: v_mov_b32_e32 v34, s18
+; GCN-NEXT: v_mov_b32_e32 v35, s19
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s12
-; GCN-NEXT: v_mov_b32_e32 v17, s13
-; GCN-NEXT: v_mov_b32_e32 v18, s14
-; GCN-NEXT: v_mov_b32_e32 v19, s15
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s12
+; GCN-NEXT: v_mov_b32_e32 v33, s13
+; GCN-NEXT: v_mov_b32_e32 v34, s14
+; GCN-NEXT: v_mov_b32_e32 v35, s15
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:16 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s8
-; GCN-NEXT: v_mov_b32_e32 v17, s9
-; GCN-NEXT: v_mov_b32_e32 v18, s10
-; GCN-NEXT: v_mov_b32_e32 v19, s11
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s8
+; GCN-NEXT: v_mov_b32_e32 v33, s9
+; GCN-NEXT: v_mov_b32_e32 v34, s10
+; GCN-NEXT: v_mov_b32_e32 v35, s11
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -1166,26 +1166,26 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; GCN-NEXT: v_mov_b32_e32 v35, s23
; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: v_mov_b32_e32 v16, s16
-; GCN-NEXT: v_mov_b32_e32 v17, s17
-; GCN-NEXT: v_mov_b32_e32 v18, s18
-; GCN-NEXT: v_mov_b32_e32 v19, s19
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
+; GCN-NEXT: s_nop 0
+; GCN-NEXT: v_mov_b32_e32 v32, s16
+; GCN-NEXT: v_mov_b32_e32 v33, s17
+; GCN-NEXT: v_mov_b32_e32 v34, s18
+; GCN-NEXT: v_mov_b32_e32 v35, s19
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s12
-; GCN-NEXT: v_mov_b32_e32 v17, s13
-; GCN-NEXT: v_mov_b32_e32 v18, s14
-; GCN-NEXT: v_mov_b32_e32 v19, s15
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s12
+; GCN-NEXT: v_mov_b32_e32 v33, s13
+; GCN-NEXT: v_mov_b32_e32 v34, s14
+; GCN-NEXT: v_mov_b32_e32 v35, s15
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:16 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_nop 0
-; GCN-NEXT: v_mov_b32_e32 v16, s8
-; GCN-NEXT: v_mov_b32_e32 v17, s9
-; GCN-NEXT: v_mov_b32_e32 v18, s10
-; GCN-NEXT: v_mov_b32_e32 v19, s11
-; GCN-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
+; GCN-NEXT: v_mov_b32_e32 v32, s8
+; GCN-NEXT: v_mov_b32_e32 v33, s9
+; GCN-NEXT: v_mov_b32_e32 v34, s10
+; GCN-NEXT: v_mov_b32_e32 v35, s11
+; GCN-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..aba29a9a1ec1a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -8244,42 +8244,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; SDAG-NEXT: s_nop 14
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; SDAG-NEXT: s_nop 12
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v20, s14
+; SDAG-NEXT: v_mov_b32_e32 v21, s15
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v22, s10
+; SDAG-NEXT: v_mov_b32_e32 v23, s11
+; SDAG-NEXT: v_mov_b32_e32 v18, s12
+; SDAG-NEXT: v_mov_b32_e32 v19, s13
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v20, s8
+; SDAG-NEXT: v_mov_b32_e32 v21, s9
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8361,42 +8362,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; AGPR-SDAG-NEXT: s_nop 14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; AGPR-SDAG-NEXT: s_nop 12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s9
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..d14fa633a6da1 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -933,10 +933,9 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 {
; GREEDY90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[2:5], v6, v7, v[0:3]
; GREEDY90A-GISEL-NEXT: s_nop 1
; GREEDY90A-GISEL-NEXT: v_mfma_f32_4x4x1f32 v[0:3], v6, v7, v[0:3]
-; GREEDY90A-GISEL-NEXT: s_nop 1
-; GREEDY90A-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GREEDY90A-GISEL-NEXT: s_nop 1
-; GREEDY90A-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GREEDY90A-GISEL-NEXT: v_mov_b32_e32 v6, 0
+; GREEDY90A-GISEL-NEXT: s_nop 3
+; GREEDY90A-GISEL-NEXT: global_store_dwordx4 v6, v[0:3], s[6:7]
; GREEDY90A-GISEL-NEXT: s_endpgm
;
; FAST90A-LABEL: test_mfma_f32_4x4x1f32:
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 8e4832f7f9e6d..db00388565b00 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -376,65 +376,65 @@ define amdgpu_kernel void @illegal_mfma_after_rewrite() #1 {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: s_mov_b32 s5, s4
-; CHECK-NEXT: v_mov_b64_e32 v[22:23], s[4:5]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[4:5]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def s[0:3]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; def v[16:19]
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_mov_b32_e32 v10, 0x7fc00000
; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; CHECK-NEXT: s_mov_b32 s0, 0x3c003c00
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[22:23], v[22:23], v[0:3]
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[0:1]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[24:25], v[24:25], v[0:3]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[0:1]
+; CHECK-NEXT: v_mov_b32_e32 v11, v10
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[24:25], v[24:25], v[4:7]
+; CHECK-NEXT: v_mov_b32_e32 v13, v10
; CHECK-NEXT: s_mov_b32 s0, 0x7e007e00
; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[4:7], v[22:23], v[22:23], v[4:7]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], s[0:1]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[24:25], v[0:3]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[6:9]
-; CHECK-NEXT: s_nop 3
-; CHECK-NEXT: v_cvt_f16_f32_e32 v20, v4
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[12:15], v[22:23], v[26:27], v[0:3]
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: v_mov_b32_e32 v8, 0x7fc00000
-; CHECK-NEXT: v_mov_b32_e32 v9, v8
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
-; CHECK-NEXT: v_mov_b32_e32 v11, v8
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[26:27], v[0:3]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], s[0:1]
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v[18:21]
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_nop 1
+; CHECK-NEXT: v_cvt_f16_f32_e32 v22, v4
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[6:9]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[24:25], v[24:25], v[10:13]
+; CHECK-NEXT: s_nop 5
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[14:17], v[24:25], v[28:29], v[0:3]
+; CHECK-NEXT: s_nop 2
; CHECK-NEXT: v_mov_b64_e32 v[0:1], 0
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[22:23], v[22:23], v[8:11]
; CHECK-NEXT: global_store_short v[0:1], v6, off
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[8:11]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[8:11]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[22:23], v[22:23], v[16:19]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[16:19], v[22:23], v[24:25], v[16:19]
-; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[24:25], v[24:25], v[18:21]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[18:21], v[24:25], v[26:27], v[18:21]
+; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_cvt_f16_f32_e32 v10, v6
; CHECK-NEXT: global_store_short v[0:1], v10, off
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[22:23], v[22:23], v[12:15]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[24:25], v[14:17]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[22:23], v[22:23], v[2:5]
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[22:23], v[22:23], v[16:19]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[24:25], v[24:25], v[2:5]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[8:11], v[24:25], v[24:25], v[18:21]
; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
; CHECK-NEXT: global_store_short v[0:1], v6, off
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[24:25], v[22:23], v[8:11]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[6:9], v[26:27], v[24:25], v[8:11]
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: global_store_short v[0:1], v20, off
+; CHECK-NEXT: global_store_short v[0:1], v22, off
; CHECK-NEXT: buffer_wbl2 sc0 sc1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_inv sc0 sc1
-; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[26:27], v[22:23], v[2:5]
+; CHECK-NEXT: v_mfma_f32_16x16x16_f16 v[2:5], v[28:29], v[24:25], v[2:5]
; CHECK-NEXT: v_cvt_f16_f32_e32 v6, v6
; CHECK-NEXT: global_store_short v[0:1], v6, off
; CHECK-NEXT: s_nop 4
>From 463d506d42371d2cfcdbcffc49a713d249c753ce Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 26 Aug 2026 10:18:38 -0500
Subject: [PATCH 05/22] Addressed review
---
llvm/lib/CodeGen/AllocationOrder.cpp | 12 +++++-------
llvm/lib/CodeGen/AllocationOrder.h | 8 ++++----
2 files changed, 9 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index 1b3d3639fbc2a..ca34a00ea5d08 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -9,7 +9,8 @@
// This file implements an allocation order for virtual registers.
//
// The preferred allocation order for a virtual register depends on allocation
-// hints and target hooks. The AllocationOrder class encapsulates all of that.
+// hints, anti-hints, and target hooks. The AllocationOrder class encapsulates
+// all of that.
//
//===----------------------------------------------------------------------===//
@@ -59,19 +60,16 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
}
});
- // Storage for filtered order (used if anti-hints cause reordering)
+ // Storage for shuffled order (used if anti-hints cause reordering)
SmallVector<MCPhysReg, 16> ShuffledOrder;
if (!AntiHintedPhysRegs.empty()) {
TRI->applyRegAllocationAntiHints(VirtReg, Order, ShuffledOrder,
AntiHintedPhysRegs, MF, &VRM, Matrix);
}
- // Use ShuffledOrder as the order if it was populated by anti-hints
- // processing
- ArrayRef<MCPhysReg> FinalOrder =
- ShuffledOrder.empty() ? Order : ShuffledOrder;
+
// Create allocation order object
- AllocationOrder AO(std::move(Hints), FinalOrder, HardHints,
+ AllocationOrder AO(std::move(Hints), Order, HardHints,
std::move(ShuffledOrder));
assert(all_of(AO.Hints,
diff --git a/llvm/lib/CodeGen/AllocationOrder.h b/llvm/lib/CodeGen/AllocationOrder.h
index 3b7c7cd4cac10..84e459c5b8e67 100644
--- a/llvm/lib/CodeGen/AllocationOrder.h
+++ b/llvm/lib/CodeGen/AllocationOrder.h
@@ -32,7 +32,7 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
const SmallVector<MCPhysReg, 16> Hints;
// Used as storage if the Order received in the constructor needs to be
// altered.
- SmallVector<MCPhysReg, 16> FilteredOrderStorage;
+ SmallVector<MCPhysReg, 16> ShuffledOrderStorage;
ArrayRef<MCPhysReg> Order;
// How far into the Order we can iterate. This is 0 if the AllocationOrder is
// constructed with HardHints = true, Order.size() otherwise. While
@@ -96,11 +96,11 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
: Hints(std::move(Hints)), Order(Order),
IterationLimit(HardHints ? 0 : static_cast<int>(Order.size())) {}
- /// Create an AllocationOrder with pre-computed FilteredOrderStorage.
+ /// Create an AllocationOrder with pre-computed ShuffledOrderStorage.
AllocationOrder(SmallVector<MCPhysReg, 16> &&Hints, ArrayRef<MCPhysReg> Order,
bool HardHints, SmallVector<MCPhysReg, 16> &&Storage)
- : Hints(std::move(Hints)), FilteredOrderStorage(std::move(Storage)),
- Order(FilteredOrderStorage.empty() ? Order : FilteredOrderStorage),
+ : Hints(std::move(Hints)), ShuffledOrderStorage(std::move(Storage)),
+ Order(ShuffledOrderStorage.empty() ? Order : ShuffledOrderStorage),
IterationLimit(HardHints ? 0 : static_cast<int>(this->Order.size())) {}
Iterator begin() const {
>From 4b809a2ba5791f96d69b6801b94e1c1e7586bed7 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 26 Aug 2026 13:50:20 -0500
Subject: [PATCH 06/22] Addressed review
---
llvm/include/llvm/CodeGen/MIRYamlMapping.h | 6 +++---
llvm/lib/CodeGen/MIRParser/MIRParser.cpp | 5 +++--
.../register-allocation-antihints-mir-print-parse.mir | 2 +-
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 05f6be547f367..142ace2cf765f 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -228,13 +228,13 @@ template <> struct MappingTraits<VirtualRegisterDefinition> {
StringValue()); // Don't print out when it's empty.
YamlIO.mapOptional("flags", Reg.RegisterFlags,
std::vector<FlowStringValue>());
- if (!YamlIO.outputting() || !Reg.AntiHints.empty())
- YamlIO.mapOptional("anti-hints", Reg.AntiHints,
- std::vector<FlowStringValue>());
// MIRPrinter sets WriteDefaultValues=true unless -simplify-mir is passed,
// so a plain mapOptional with an empty default would still emit the keys
// and change every existing test's output.
// Skip the call on output when empty to keep them off entirely.
+ if (!YamlIO.outputting() || !Reg.AntiHints.empty())
+ YamlIO.mapOptional("anti-hints", Reg.AntiHints,
+ std::vector<FlowStringValue>());
if (!YamlIO.outputting() || !Reg.SplitFrom.Value.empty())
YamlIO.mapOptional("split-from", Reg.SplitFrom, StringValue());
if (!YamlIO.outputting() || !Reg.AssignedPhys.Value.empty())
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 8783078485877..febb010bb4d09 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -781,10 +781,11 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS,
if (parseRegisterReference(PFS, AntiHintReg, AntiHintValue.Value, Error))
return error(Error, AntiHintValue.SourceRange);
- if (!AntiHintReg.isVirtual())
+ if (!AntiHintReg.isVirtual()) {
return error(AntiHintValue.SourceRange.Start,
- Twine("anti-hint '") + AntiHintValue.Value +
+ "anti-hint '" + Twine(AntiHintValue.Value) +
"' must be a virtual register");
+ }
Info.AntiHints.push_back(AntiHintReg);
}
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir b/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
index 574fe61c416de..12e15c3ef55b8 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/register-allocation-antihints-mir-print-parse.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=none -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=none -o - %s | FileCheck %s
---
name: single_anti_hint
>From a018b6c8a3ee5a4afeb50680faa2d0db5525cea7 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sun, 30 Aug 2026 15:00:47 -0500
Subject: [PATCH 07/22] Addressed review: Removed ShuffledOrderStorage
---
.../include/llvm/CodeGen/TargetRegisterInfo.h | 13 +++---
llvm/lib/CodeGen/AllocationOrder.cpp | 31 +++++++------
llvm/lib/CodeGen/AllocationOrder.h | 44 +++++++++++--------
llvm/lib/CodeGen/TargetRegisterInfo.cpp | 25 +++++------
.../unittests/CodeGen/AllocationOrderTest.cpp | 21 ++++-----
5 files changed, 69 insertions(+), 65 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index e0c358d25a087..bdb2a4d0363b5 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -853,13 +853,12 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
}
/// Apply anti-hints to the allocation order.
- virtual void
- applyRegAllocationAntiHints(Register VirtReg, ArrayRef<MCPhysReg> &Order,
- SmallVectorImpl<MCPhysReg> &OrderStorage,
- SmallVectorImpl<MCPhysReg> &AntiHints,
- const MachineFunction &MF,
- const VirtRegMap *VRM = nullptr,
- const LiveRegMatrix *Matrix = nullptr) const;
+ virtual void applyRegAllocationAntiHints(
+ Register VirtReg, ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
+ SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
+ const VirtRegMap *VRM = nullptr,
+ const LiveRegMatrix *Matrix = nullptr) const;
/// Allow the target to reverse allocation order of local live ranges. This
/// will generally allocate shorter local live ranges first. For targets with
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index ca34a00ea5d08..2367095ce23df 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -34,14 +34,20 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
const TargetRegisterInfo *TRI = &VRM.getTargetRegInfo();
const MachineRegisterInfo &MRI = MF.getRegInfo();
auto Order = RegClassInfo.getOrder(MF.getRegInfo().getRegClass(VirtReg));
- SmallVector<MCPhysReg, 16> Hints;
- bool HardHints =
- TRI->getRegAllocationHints(VirtReg, Order, Hints, MF, &VRM, Matrix);
+
+ // HintsAndCustomOrder holds Hints first followed by the shuffled order if the
+ // anti-hints shuffle it.
+ SmallVector<MCPhysReg, 16> HintsAndCustomOrder;
+
+ // Get hints
+ bool HardHints = TRI->getRegAllocationHints(
+ VirtReg, Order, HintsAndCustomOrder, MF, &VRM, Matrix);
+ const int NumHints = static_cast<int>(HintsAndCustomOrder.size());
LLVM_DEBUG({
- if (!Hints.empty()) {
+ if (NumHints) {
dbgs() << "hints:";
- for (MCPhysReg Hint : Hints)
+ for (MCPhysReg Hint : HintsAndCustomOrder)
dbgs() << ' ' << printReg(Hint, TRI);
dbgs() << '\n';
}
@@ -60,19 +66,18 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
}
});
- // Storage for shuffled order (used if anti-hints cause reordering)
- SmallVector<MCPhysReg, 16> ShuffledOrder;
-
if (!AntiHintedPhysRegs.empty()) {
- TRI->applyRegAllocationAntiHints(VirtReg, Order, ShuffledOrder,
- AntiHintedPhysRegs, MF, &VRM, Matrix);
+ HintsAndCustomOrder.reserve(NumHints + Order.size());
+ TRI->applyRegAllocationAntiHints(VirtReg, Order, HintsAndCustomOrder,
+ NumHints, AntiHintedPhysRegs, MF, &VRM,
+ Matrix);
}
// Create allocation order object
- AllocationOrder AO(std::move(Hints), Order, HardHints,
- std::move(ShuffledOrder));
+ AllocationOrder AO(std::move(HintsAndCustomOrder), NumHints, Order,
+ HardHints);
- assert(all_of(AO.Hints,
+ assert(all_of(AO.hints(),
[&](MCPhysReg Hint) { return is_contained(AO.Order, Hint); }) &&
"Target hint is outside allocation order.");
return AO;
diff --git a/llvm/lib/CodeGen/AllocationOrder.h b/llvm/lib/CodeGen/AllocationOrder.h
index 84e459c5b8e67..643c927103802 100644
--- a/llvm/lib/CodeGen/AllocationOrder.h
+++ b/llvm/lib/CodeGen/AllocationOrder.h
@@ -29,10 +29,11 @@ class VirtRegMap;
class LiveRegMatrix;
class LLVM_LIBRARY_VISIBILITY AllocationOrder {
- const SmallVector<MCPhysReg, 16> Hints;
- // Used as storage if the Order received in the constructor needs to be
- // altered.
- SmallVector<MCPhysReg, 16> ShuffledOrderStorage;
+ // The Hints occupy [0, NumHints). If the Order received in the constructor
+ // needed to be shuffled, the shuffled copy is stored right after them, at
+ // [NumHints, end).
+ const SmallVector<MCPhysReg, 16> HintsAndCustomOrder;
+ const int NumHints;
ArrayRef<MCPhysReg> Order;
// How far into the Order we can iterate. This is 0 if the AllocationOrder is
// constructed with HardHints = true, Order.size() otherwise. While
@@ -43,6 +44,10 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
// IterationLimit defines an invalid iterator position.
const int IterationLimit;
+ ArrayRef<MCPhysReg> hints() const {
+ return ArrayRef<MCPhysReg>(HintsAndCustomOrder).take_front(NumHints);
+ }
+
public:
/// Forward iterator for an AllocationOrder.
class Iterator final {
@@ -58,7 +63,7 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
/// Return the next physical register in the allocation order.
MCRegister operator*() const {
if (Pos < 0)
- return AO.Hints.end()[Pos];
+ return AO.HintsAndCustomOrder[AO.NumHints + Pos];
assert(Pos < AO.IterationLimit);
return AO.Order[Pos];
}
@@ -89,23 +94,26 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
const RegisterClassInfo &RegClassInfo,
const LiveRegMatrix *Matrix);
+ /// Create an AllocationOrder from HintsAndCustomOrder that contains NumHints
+ /// Hints optionally followed by a reordered Order. If no reordered copy is
+ /// present use Order as-is.
+ AllocationOrder(SmallVector<MCPhysReg, 16> &&HintsAndCustomOrder,
+ int NumHints, ArrayRef<MCPhysReg> Order, bool HardHints)
+ : HintsAndCustomOrder(std::move(HintsAndCustomOrder)), NumHints(NumHints),
+ Order(static_cast<int>(this->HintsAndCustomOrder.size()) > NumHints
+ ? ArrayRef<MCPhysReg>(this->HintsAndCustomOrder)
+ .drop_front(NumHints)
+ : Order),
+ IterationLimit(HardHints ? 0 : static_cast<int>(this->Order.size())) {}
+
/// Create an AllocationOrder given the Hints, Order, and HardHints values.
/// Use the create method above - the ctor is for unittests.
AllocationOrder(SmallVector<MCPhysReg, 16> &&Hints, ArrayRef<MCPhysReg> Order,
bool HardHints)
- : Hints(std::move(Hints)), Order(Order),
- IterationLimit(HardHints ? 0 : static_cast<int>(Order.size())) {}
+ : AllocationOrder(std::move(Hints), static_cast<int>(Hints.size()), Order,
+ HardHints) {}
- /// Create an AllocationOrder with pre-computed ShuffledOrderStorage.
- AllocationOrder(SmallVector<MCPhysReg, 16> &&Hints, ArrayRef<MCPhysReg> Order,
- bool HardHints, SmallVector<MCPhysReg, 16> &&Storage)
- : Hints(std::move(Hints)), ShuffledOrderStorage(std::move(Storage)),
- Order(ShuffledOrderStorage.empty() ? Order : ShuffledOrderStorage),
- IterationLimit(HardHints ? 0 : static_cast<int>(this->Order.size())) {}
-
- Iterator begin() const {
- return Iterator(*this, -(static_cast<int>(Hints.size())));
- }
+ Iterator begin() const { return Iterator(*this, -NumHints); }
Iterator end() const { return Iterator(*this, IterationLimit); }
@@ -126,7 +134,7 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
assert(!Reg.isPhysical() ||
Reg.id() <
static_cast<uint32_t>(std::numeric_limits<MCPhysReg>::max()));
- return Reg.isPhysical() && is_contained(Hints, Reg.id());
+ return Reg.isPhysical() && is_contained(hints(), Reg.id());
}
};
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index fd696297e35d9..44defb5e455a8 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -444,8 +444,8 @@ bool TargetRegisterInfo::getRegAllocationHints(
}
void TargetRegisterInfo::applyRegAllocationAntiHints(
- Register VirtReg, ArrayRef<MCPhysReg> &Order,
- SmallVectorImpl<MCPhysReg> &OrderStorage,
+ Register VirtReg, ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
@@ -458,29 +458,24 @@ void TargetRegisterInfo::applyRegAllocationAntiHints(
});
};
- // Copy order into storage
- OrderStorage.clear();
- OrderStorage.assign(Order.begin(), Order.end());
+ HintsAndCustomOrder.truncate(NumHints);
+ HintsAndCustomOrder.append(Order.begin(), Order.end());
// Partition non-anti-hinted register go first
- auto PartionPoint =
- std::stable_partition(OrderStorage.begin(), OrderStorage.end(),
- [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
-
- Order = OrderStorage;
+ auto *PartitionPoint = std::stable_partition(
+ HintsAndCustomOrder.begin() + NumHints, HintsAndCustomOrder.end(),
+ [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
- // print the details
LLVM_DEBUG({
size_t NonAntiHintedCount =
- std::distance(OrderStorage.begin(), PartionPoint);
- size_t AntiHintedCount = std::distance(PartionPoint, OrderStorage.end());
+ std::distance(HintsAndCustomOrder.begin() + NumHints, PartitionPoint);
+ size_t AntiHintedCount =
+ std::distance(PartitionPoint, HintsAndCustomOrder.end());
dbgs() << "Added " << NonAntiHintedCount
<< " non-anti-hinted registers first\n"
<< "Added " << AntiHintedCount
<< " anti-hinted registers at the end\n";
});
-
- return;
}
bool TargetRegisterInfo::isCalleeSavedPhysReg(
diff --git a/llvm/unittests/CodeGen/AllocationOrderTest.cpp b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
index ec02579f55c58..aa636b6673b1e 100644
--- a/llvm/unittests/CodeGen/AllocationOrderTest.cpp
+++ b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
@@ -117,26 +117,23 @@ TEST(AllocationOrderTest, IsHintTest) {
EXPECT_EQ(V, 5U);
}
-TEST(AllocationOrderTest, StorageOverridesOrder) {
- SmallVector<MCPhysReg, 16> Hints = {1, 2};
+TEST(AllocationOrderTest, AntiHintShuffleOrderOverridesOrder) {
+ SmallVector<MCPhysReg, 16> HintsAndCustomOrder = {1, 2, 5, 3, 4};
SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
- SmallVector<MCPhysReg, 16> Storage = {5, 3, 4};
- AllocationOrder O(std::move(Hints), Order, false, std::move(Storage));
+ AllocationOrder O(std::move(HintsAndCustomOrder), 2, Order, false);
EXPECT_EQ((std::vector<MCPhysReg>{1, 2, 5, 3, 4}), loadOrder(O));
}
-TEST(AllocationOrderTest, EmptyStorageFallsBackToOrder) {
- SmallVector<MCPhysReg, 16> Hints = {1, 2};
+TEST(AllocationOrderTest, EmptyAntiHintedCustomOrderFallsBackToOrder) {
+ SmallVector<MCPhysReg, 16> HintsAndCustomOrder = {1, 2};
SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
- SmallVector<MCPhysReg, 16> Storage;
- AllocationOrder O(std::move(Hints), Order, false, std::move(Storage));
+ AllocationOrder O(std::move(HintsAndCustomOrder), 2, Order, false);
EXPECT_EQ((std::vector<MCPhysReg>{1, 2, 3, 4, 5}), loadOrder(O));
}
-TEST(AllocationOrderTest, StorageHardHints) {
- SmallVector<MCPhysReg, 16> Hints = {1, 2};
+TEST(AllocationOrderTest, HardHintsIgnoreAntiHintShuffleOrder) {
+ SmallVector<MCPhysReg, 16> HintsAndCustomOrder = {1, 2, 5, 3, 4};
SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
- SmallVector<MCPhysReg, 16> Storage = {5, 3, 4};
- AllocationOrder O(std::move(Hints), Order, true, std::move(Storage));
+ AllocationOrder O(std::move(HintsAndCustomOrder), 2, Order, true);
EXPECT_EQ((std::vector<MCPhysReg>{1, 2}), loadOrder(O));
}
>From 5a1ffce4766e26eb752daaf0e1edade9fdeddf85 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sun, 30 Aug 2026 21:27:17 -0500
Subject: [PATCH 08/22] Addressed review: Removed OrderStorage
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 25 +++++++++++------------
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 4 ++--
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 97fca7ca20a26..b83922edabdd0 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4224,8 +4224,8 @@ bool SIRegisterInfo::shouldApplyAntiHints(
}
void SIRegisterInfo::applyRegAllocationAntiHints(
- Register VirtReg, ArrayRef<MCPhysReg> &Order,
- SmallVectorImpl<MCPhysReg> &OrderStorage,
+ Register VirtReg, ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
@@ -4266,9 +4266,8 @@ void SIRegisterInfo::applyRegAllocationAntiHints(
return Highest < MaxVGPRsForCurrentOccupancy;
};
- // Copy order.
- OrderStorage.clear();
- OrderStorage.assign(Order.begin(), Order.end());
+ HintsAndCustomOrder.truncate(NumHints);
+ HintsAndCustomOrder.append(Order.begin(), Order.end());
// Helper to check if a register overlaps with any anti-hint.
auto isAntiHinted = [&](MCPhysReg Reg) {
@@ -4278,21 +4277,21 @@ void SIRegisterInfo::applyRegAllocationAntiHints(
};
// Find the cutoff point for the current occupancy VGPR budget.
- auto *BeyondBudgetStart = llvm::find_if(
- OrderStorage, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
+ auto *BeyondBudgetStart =
+ llvm::find_if(llvm::drop_begin(HintsAndCustomOrder, NumHints),
+ [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
// Only shuffle within the current occupancy VGPR budget.
- auto *PartitionPoint =
- std::stable_partition(OrderStorage.begin(), BeyondBudgetStart,
- [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
+ auto *PartitionPoint = std::stable_partition(
+ HintsAndCustomOrder.begin() + NumHints, BeyondBudgetStart,
+ [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
- Order = OrderStorage;
LLVM_DEBUG({
size_t NonAntiHintedCount =
- std::distance(OrderStorage.begin(), PartitionPoint);
+ std::distance(HintsAndCustomOrder.begin() + NumHints, PartitionPoint);
size_t AntiHintedCount = std::distance(PartitionPoint, BeyondBudgetStart);
size_t BeyondBudgetCount =
- std::distance(BeyondBudgetStart, OrderStorage.end());
+ std::distance(BeyondBudgetStart, HintsAndCustomOrder.end());
dbgs() << "Added " << NonAntiHintedCount
<< " non-anti-hinted registers first\n"
<< "Added " << AntiHintedCount
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 61fbf90d84488..7765849ab804b 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -374,8 +374,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
unsigned &MaxVGPRsForCurrentOccupancy) const;
void applyRegAllocationAntiHints(
- Register VirtReg, ArrayRef<MCPhysReg> &Order,
- SmallVectorImpl<MCPhysReg> &OrderStorage,
+ Register VirtReg, ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
const VirtRegMap *VRM = nullptr,
const LiveRegMatrix *Matrix = nullptr) const override;
>From 17742ac7d881877907d769a0ca04bb9d0007517a Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Tue, 1 Sep 2026 14:02:53 -0500
Subject: [PATCH 09/22] Addressed review: Added test
---
.../AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 406 +++++++++++++-----
1 file changed, 300 insertions(+), 106 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
index 5d698f533acf2..6145c11dbc54b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
@@ -1,12 +1,16 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,machineverifier,virtregrewriter,post-RA-hazard-rec %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=greedy,machineverifier,virtregrewriter,post-RA-hazard-rec %s -o - | FileCheck %s --check-prefixes=AH,GFX942
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=greedy,machineverifier,virtregrewriter,post-RA-hazard-rec %s -o - | FileCheck %s --check-prefixes=AH,GFX90A
--- |
define amdgpu_kernel void @anti_hints_present() #0 { ret void }
define amdgpu_kernel void @anti_hints_absent() #0 { ret void }
define amdgpu_kernel void @anti_hints_suppressed_by_target_budget() #0 { ret void }
define amdgpu_kernel void @anti_hints_suppressed_by_current_budget() #0 { ret void }
+ define amdgpu_kernel void @anti_hints_unified_agpr_headroom () #1 { ret void }
+ define amdgpu_kernel void @anti_hints_unified_agpr_budget () #1 { ret void }
attributes #0 = { nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+ attributes #1 = { nounwind "frame-pointer"="none"}
...
---
@@ -43,30 +47,30 @@ liveins:
body: |
bb.0:
liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
- ; CHECK-LABEL: name: anti_hints_present
- ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
+ ; AH-LABEL: name: anti_hints_present
+ ; AH: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; AH-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; AH-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vgpr_32 = COPY $vgpr4
%2:vgpr_32 = COPY $vgpr5
@@ -127,33 +131,61 @@ liveins:
body: |
bb.0:
liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5
- ; CHECK-LABEL: name: anti_hints_absent
- ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
- ; CHECK-NEXT: S_NOP 4
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
- ; CHECK-NEXT: S_NOP 5
- ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: S_NOP 6
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ; GFX942-LABEL: name: anti_hints_absent
+ ; GFX942: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NEXT: S_NOP 4
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NEXT: S_NOP 5
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: S_NOP 6
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
+ ;
+ ; GFX90A-LABEL: name: anti_hints_absent
+ ; GFX90A: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: S_NOP 12
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NEXT: S_NOP 13
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: S_NOP 14
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vgpr_32 = COPY $vgpr4
%2:vgpr_32 = COPY $vgpr5
@@ -220,33 +252,61 @@ liveins:
body: |
bb.0:
liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
- ; CHECK-LABEL: name: anti_hints_suppressed_by_target_budget
- ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
- ; CHECK-NEXT: S_NOP 4
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
- ; CHECK-NEXT: S_NOP 5
- ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: S_NOP 6
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-LABEL: name: anti_hints_suppressed_by_target_budget
+ ; GFX942: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NEXT: S_NOP 4
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NEXT: S_NOP 5
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: S_NOP 6
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ ;
+ ; GFX90A-LABEL: name: anti_hints_suppressed_by_target_budget
+ ; GFX90A: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: S_NOP 12
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NEXT: S_NOP 13
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: S_NOP 14
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
%20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55
%21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59
%22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63
@@ -316,33 +376,61 @@ liveins:
body: |
bb.0:
liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
- ; CHECK-LABEL: name: anti_hints_suppressed_by_current_budget
- ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
- ; CHECK-NEXT: S_NOP 4
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
- ; CHECK-NEXT: S_NOP 5
- ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
- ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: S_NOP 6
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
- ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
- ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-LABEL: name: anti_hints_suppressed_by_current_budget
+ ; GFX942: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NEXT: S_NOP 4
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NEXT: S_NOP 5
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: S_NOP 6
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
+ ;
+ ; GFX90A-LABEL: name: anti_hints_suppressed_by_current_budget
+ ; GFX90A: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: S_NOP 12
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NEXT: S_NOP 13
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: S_NOP 14
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63
%20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55
%21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59
%22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63
@@ -371,3 +459,109 @@ body: |
%19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit %20, implicit %21, implicit %22
...
+
+---
+name: anti_hints_unified_agpr_headroom
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 6
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vreg_128_align2 }
+ - { id: 2, class: vreg_128_align2 }
+ - { id: 3, class: vreg_128_align2 }
+ - { id: 4, class: vreg_512_align2, anti-hints: [ '%8' ] }
+ - { id: 5, class: vreg_128_align2 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: areg_256_align2 }
+ - { id: 8, class: vreg_512_align2, anti-hints: [ '%4' ] }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31' }
+ - { reg: '$vgpr32_vgpr33_vgpr34_vgpr35' }
+ - { reg: '$vgpr36_vgpr37_vgpr38_vgpr39' }
+ - { reg: '$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ ; AH-LABEL: name: anti_hints_unified_agpr_headroom
+ ; AH: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: dead renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr4_vgpr5, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = DS_READ_B128_gfx9 renamable $vgpr0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr44_vgpr45_vgpr46_vgpr47 = DS_READ_B128_gfx9 renamable $vgpr0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr48_vgpr49_vgpr50_vgpr51 = DS_READ_B128_gfx9 renamable $vgpr0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr52_vgpr53_vgpr54_vgpr55 = DS_READ_B128_gfx9 renamable $vgpr0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, implicit killed renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vreg_512_align2 = COPY $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ %5:vreg_128_align2 = COPY $vgpr32_vgpr33_vgpr34_vgpr35
+ %6:vreg_128_align2 = COPY $vgpr36_vgpr37_vgpr38_vgpr39
+ %7:areg_256_align2 = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+ %4:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %0.sub0_sub1, %1.sub0_sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ undef %8.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %5, implicit %6, implicit %7, implicit %8
+...
+
+---
+name: anti_hints_unified_agpr_budget
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 6
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vreg_128_align2 }
+ - { id: 2, class: vreg_128_align2 }
+ - { id: 3, class: vreg_128_align2 }
+ - { id: 4, class: vreg_512_align2, anti-hints: [ '%8' ] }
+ - { id: 5, class: vreg_128_align2 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: areg_512_align2 }
+ - { id: 8, class: vreg_512_align2, anti-hints: [ '%4' ] }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31' }
+ - { reg: '$vgpr32_vgpr33_vgpr34_vgpr35' }
+ - { reg: '$vgpr36_vgpr37_vgpr38_vgpr39' }
+ - { reg: '$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ ; AH-LABEL: name: anti_hints_unified_agpr_budget
+ ; AH: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: dead renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr4_vgpr5, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = DS_READ_B128_gfx9 renamable $vgpr0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr44_vgpr45_vgpr46_vgpr47 = DS_READ_B128_gfx9 renamable $vgpr0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr48_vgpr49_vgpr50_vgpr51 = DS_READ_B128_gfx9 renamable $vgpr0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: renamable $vgpr52_vgpr53_vgpr54_vgpr55 = DS_READ_B128_gfx9 renamable $vgpr0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vreg_512_align2 = COPY $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ %5:vreg_128_align2 = COPY $vgpr32_vgpr33_vgpr34_vgpr35
+ %6:vreg_128_align2 = COPY $vgpr36_vgpr37_vgpr38_vgpr39
+ %7:areg_512_align2 = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ %4:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %0.sub0_sub1, %1.sub0_sub1, %4, 0, 0, 0, implicit $mode, implicit $exec
+ undef %8.sub0_sub1_sub2_sub3:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub4_sub5_sub6_sub7:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ %8.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %5, implicit %6, implicit %7, implicit %8
+...
\ No newline at end of file
>From 47d8d5da1bf5ba803e7d87b24b64d249d509f30b Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Tue, 1 Sep 2026 14:23:36 -0500
Subject: [PATCH 10/22] Addressed reveiw
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 24 ++++++++++----
.../AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 33 +++++++++++++------
2 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index b83922edabdd0..93c9b1f8e948c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -23,7 +23,6 @@
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
-#include "llvm/MC/MCRegister.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
@@ -4199,8 +4198,11 @@ bool SIRegisterInfo::shouldApplyAntiHints(
if (CurrentOccupancy == 1)
return true;
- // Set max VGPRs for target and current occupancy to early bail out if we are
- // close to the limit.
+ // Do not apply anti-hints if we are reaching close to the VGPR budget. For
+ // target occupancy, the 80% cutoff is a conservative: anti-hints are disabled
+ // early enough that later registers still have headroom to stay at target
+ // occupancy. For current occupancy, the 95% cutoff margin is used to apply
+ // anti-hints close to the limit of the current occupancy budget.
unsigned MaxVGPRsCutOffForTargetOccupancy =
(ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100;
unsigned MaxVGPRsCutOffForCurrentOccupancy =
@@ -4261,9 +4263,19 @@ void SIRegisterInfo::applyRegAllocationAntiHints(
(!isVGPRClass(RC) && !isAGPRClass(RC) && !isVectorSuperClass(RC)))
return true;
- unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
- unsigned Highest = getHWRegIndex(Reg) + NumRegs - 1;
- return Highest < MaxVGPRsForCurrentOccupancy;
+ unsigned RegEndIndex =
+ getHWRegIndex(Reg) + divideCeil(getRegSizeInBits(*RC), 32);
+
+ unsigned MaxVGPR = NumVGPRs;
+ unsigned MaxAGPR = NumAGPRs;
+ if (isAGPRClass(RC))
+ MaxAGPR = std::max(MaxAGPR, RegEndIndex);
+ else
+ MaxVGPR = std::max(MaxVGPR, RegEndIndex);
+
+ return static_cast<unsigned>(AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(),
+ MaxAGPR, MaxVGPR)) <=
+ MaxVGPRsForCurrentOccupancy;
};
HintsAndCustomOrder.truncate(NumHints);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
index 6145c11dbc54b..2eec9fe8c8eb2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
@@ -541,15 +541,28 @@ liveins:
body: |
bb.0:
liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
- ; AH-LABEL: name: anti_hints_unified_agpr_budget
- ; AH: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
- ; AH-NEXT: {{ $}}
- ; AH-NEXT: dead renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr4_vgpr5, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
- ; AH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = DS_READ_B128_gfx9 renamable $vgpr0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
- ; AH-NEXT: renamable $vgpr44_vgpr45_vgpr46_vgpr47 = DS_READ_B128_gfx9 renamable $vgpr0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
- ; AH-NEXT: renamable $vgpr48_vgpr49_vgpr50_vgpr51 = DS_READ_B128_gfx9 renamable $vgpr0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
- ; AH-NEXT: renamable $vgpr52_vgpr53_vgpr54_vgpr55 = DS_READ_B128_gfx9 renamable $vgpr0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
- ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+ ; GFX942-LABEL: name: anti_hints_unified_agpr_budget
+ ; GFX942: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: dead renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr4_vgpr5, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: S_NOP 10
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = DS_READ_B128_gfx9 renamable $vgpr0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr20_vgpr21_vgpr22_vgpr23 = DS_READ_B128_gfx9 renamable $vgpr0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 renamable $vgpr0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ;
+ ; GFX90A-LABEL: name: anti_hints_unified_agpr_budget
+ ; GFX90A: liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35, $vgpr36_vgpr37_vgpr38_vgpr39, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: dead renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = contract V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr4_vgpr5, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: S_NOP 15
+ ; GFX90A-NEXT: S_NOP 2
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = DS_READ_B128_gfx9 renamable $vgpr0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr20_vgpr21_vgpr22_vgpr23 = DS_READ_B128_gfx9 renamable $vgpr0, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 renamable $vgpr0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
%2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
@@ -564,4 +577,4 @@ body: |
%8.sub8_sub9_sub10_sub11:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
%8.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %5, implicit %6, implicit %7, implicit %8
-...
\ No newline at end of file
+...
>From 82e68bdd3c0dd3f2e24ad168103edb2f4ffbdbc0 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Fri, 4 Sep 2026 22:29:10 -0500
Subject: [PATCH 11/22] Addressed review and added target overrideable
filterAndSort
---
.../llvm/CodeGen/MachineRegisterInfo.h | 9 ++--
.../include/llvm/CodeGen/TargetRegisterInfo.h | 15 +++++--
llvm/lib/CodeGen/AllocationOrder.cpp | 30 +++++++------
llvm/lib/CodeGen/AllocationOrder.h | 10 ++---
llvm/lib/CodeGen/MachineRegisterInfo.cpp | 26 ++++++------
llvm/lib/CodeGen/TargetRegisterInfo.cpp | 42 ++++++++++++-------
.../unittests/CodeGen/AllocationOrderTest.cpp | 4 +-
7 files changed, 81 insertions(+), 55 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 890355febbb28..69d14fd11507f 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -922,7 +922,7 @@ class MachineRegisterInfo {
"Anti-hints and anti-hint targets are only for virtual registers");
AntiHintRegs.grow(VReg);
SmallVector<Register, 4> &AntiHints = AntiHintRegs[VReg];
- // Avoid duplicates
+ // Avoid duplicates.
if (!is_contained(AntiHints, AntiHintVReg))
AntiHints.push_back(AntiHintVReg);
}
@@ -959,11 +959,10 @@ class MachineRegisterInfo {
return is_contained(AntiHints, AntiHintVReg);
}
- /// Get the set of physical registers to avoid.
+ /// Get the BitVector of register units to avoid in anti-hints.
/// VRM is the current virtual register map showing allocations made so far.
- void getPhysRegAntiHints(Register VReg,
- SmallVectorImpl<MCPhysReg> &PhysAntiHints,
- const VirtRegMap &VRM) const;
+ void getBitVecRegAntiHints(Register VReg, BitVector &AntiHintedRegUnits,
+ const VirtRegMap &VRM) const;
/// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the
/// specified register as undefined which causes the DBG_VALUE to be
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index bdb2a4d0363b5..9de9e644effda 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -852,12 +852,21 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
// Do nothing.
}
+ /// Return true if Reg overlaps one of the anti-hinted register units.
+ bool isAntiHintedReg(MCPhysReg Reg,
+ const BitVector &AntiHintedRegUnits) const;
+
/// Apply anti-hints to the allocation order.
- virtual void applyRegAllocationAntiHints(
+ void applyRegAllocationAntiHints(
Register VirtReg, ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
- SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
- const VirtRegMap *VRM = nullptr,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
+ const LiveRegMatrix *Matrix = nullptr) const;
+
+ /// Custom reordering of the allocation order.
+ virtual void filterAndSortForAntiHintedRegs(
+ Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
const LiveRegMatrix *Matrix = nullptr) const;
/// Allow the target to reverse allocation order of local live ranges. This
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index 2367095ce23df..ac4485c849a00 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -15,6 +15,7 @@
//===----------------------------------------------------------------------===//
#include "AllocationOrder.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
@@ -35,15 +36,16 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
const MachineRegisterInfo &MRI = MF.getRegInfo();
auto Order = RegClassInfo.getOrder(MF.getRegInfo().getRegClass(VirtReg));
- // HintsAndCustomOrder holds Hints first followed by the shuffled order if the
- // anti-hints shuffle it.
+ // HintsAndCustomOrder holds Hints first followed by the custom order if the
+ // anti-hints reorders it.
SmallVector<MCPhysReg, 16> HintsAndCustomOrder;
- // Get hints
+ // Get Hints.
bool HardHints = TRI->getRegAllocationHints(
VirtReg, Order, HintsAndCustomOrder, MF, &VRM, Matrix);
const int NumHints = static_cast<int>(HintsAndCustomOrder.size());
+ // HintsAndCustomOrder only holds Hints (custom order is not added yet).
LLVM_DEBUG({
if (NumHints) {
dbgs() << "hints:";
@@ -53,27 +55,29 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
}
});
- // Get anti-hints
- SmallVector<MCPhysReg, 16> AntiHintedPhysRegs;
- MRI.getPhysRegAntiHints(VirtReg, AntiHintedPhysRegs, VRM);
+ // Get anti-hints.
+ BitVector AntiHintedRegUnits;
+ MRI.getBitVecRegAntiHints(VirtReg, AntiHintedRegUnits, VRM);
LLVM_DEBUG({
- if (!AntiHintedPhysRegs.empty()) {
+ if (AntiHintedRegUnits.any()) {
dbgs() << "anti-hints:";
- for (MCPhysReg AntiHint : AntiHintedPhysRegs)
- dbgs() << ' ' << printReg(AntiHint, TRI);
+ for (Register AntiHintVReg : MRI.getRegAllocationAntiHints(VirtReg)) {
+ if (!VRM.hasPhys(AntiHintVReg))
+ continue;
+ dbgs() << ' ' << printReg(VRM.getPhys(AntiHintVReg), TRI);
+ }
dbgs() << '\n';
}
});
- if (!AntiHintedPhysRegs.empty()) {
+ if (AntiHintedRegUnits.any()) {
HintsAndCustomOrder.reserve(NumHints + Order.size());
TRI->applyRegAllocationAntiHints(VirtReg, Order, HintsAndCustomOrder,
- NumHints, AntiHintedPhysRegs, MF, &VRM,
- Matrix);
+ NumHints, AntiHintedRegUnits, MF, Matrix);
}
- // Create allocation order object
+ // Create allocation order object.
AllocationOrder AO(std::move(HintsAndCustomOrder), NumHints, Order,
HardHints);
diff --git a/llvm/lib/CodeGen/AllocationOrder.h b/llvm/lib/CodeGen/AllocationOrder.h
index 643c927103802..09d8fd68de17b 100644
--- a/llvm/lib/CodeGen/AllocationOrder.h
+++ b/llvm/lib/CodeGen/AllocationOrder.h
@@ -29,9 +29,9 @@ class VirtRegMap;
class LiveRegMatrix;
class LLVM_LIBRARY_VISIBILITY AllocationOrder {
- // The Hints occupy [0, NumHints). If the Order received in the constructor
- // needed to be shuffled, the shuffled copy is stored right after them, at
- // [NumHints, end).
+ // Used as storage for both Hints and CustomOrder if the Order received in the
+ // constructor needs to be altered. [0, NumHints) contains regular hints. If a
+ // custom order is present, [NumHints, end) contains the custom order.
const SmallVector<MCPhysReg, 16> HintsAndCustomOrder;
const int NumHints;
ArrayRef<MCPhysReg> Order;
@@ -95,8 +95,8 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
const LiveRegMatrix *Matrix);
/// Create an AllocationOrder from HintsAndCustomOrder that contains NumHints
- /// Hints optionally followed by a reordered Order. If no reordered copy is
- /// present use Order as-is.
+ /// Hints optionally followed by a custom order. When that custom order is
+ /// present it becomes the allocation order otherwise Order is used as-is.
AllocationOrder(SmallVector<MCPhysReg, 16> &&HintsAndCustomOrder,
int NumHints, ArrayRef<MCPhysReg> Order, bool HardHints)
: HintsAndCustomOrder(std::move(HintsAndCustomOrder)), NumHints(NumHints),
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index 3918c169b978c..61fae6d3fa281 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -718,21 +718,23 @@ void MachineRegisterInfo::updateDbgUsersToReg(
}
}
-void MachineRegisterInfo::getPhysRegAntiHints(
- Register VReg, SmallVectorImpl<MCPhysReg> &PhysAntiHints,
- const VirtRegMap &VRM) const {
+void MachineRegisterInfo::getBitVecRegAntiHints(Register VReg,
+ BitVector &AntiHintedRegUnits,
+ const VirtRegMap &VRM) const {
assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
if (!AntiHintRegs.inBounds(VReg))
return;
- const SmallVector<Register, 4> &AntiHints = AntiHintRegs[VReg];
-
- for (Register AntiHintVReg : AntiHints) {
- // Check if the anti-hinted register has been allocated
- if (VRM.hasPhys(AntiHintVReg)) {
- MCPhysReg PhysReg = VRM.getPhys(AntiHintVReg);
- if (!is_contained(PhysAntiHints, PhysReg))
- PhysAntiHints.push_back(PhysReg);
- }
+ auto *TRI = getTargetRegisterInfo();
+ for (Register AntiHintVReg : AntiHintRegs[VReg]) {
+ // Check if the anti-hinted register has been allocated.
+ if (!VRM.hasPhys(AntiHintVReg))
+ continue;
+ // Delay until first allocated anti-hinted register so unused cases keep
+ // default empty BitVector.
+ if (AntiHintedRegUnits.empty())
+ AntiHintedRegUnits.resize(TRI->getNumRegUnits());
+ for (MCRegUnit Unit : TRI->regunits(VRM.getPhys(AntiHintVReg)))
+ AntiHintedRegUnits.set(static_cast<unsigned>(Unit));
}
}
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index 44defb5e455a8..f6ec5c9ac46e0 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -443,34 +443,46 @@ bool TargetRegisterInfo::getRegAllocationHints(
return false;
}
+bool TargetRegisterInfo::isAntiHintedReg(
+ MCPhysReg Reg, const BitVector &AntiHintedRegUnits) const {
+ return llvm::any_of(regunits(Reg), [&](MCRegUnit Unit) {
+ return AntiHintedRegUnits.test(static_cast<unsigned>(Unit));
+ });
+}
+
void TargetRegisterInfo::applyRegAllocationAntiHints(
Register VirtReg, ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
- SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
- const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
+ const LiveRegMatrix *Matrix) const {
- if (AntiHints.empty() || !VRM)
+ if (AntiHintedRegUnits.none())
return;
- auto isAntiHinted = [&](MCPhysReg Reg) {
- return llvm::any_of(AntiHints, [&](MCPhysReg AntiHint) {
- return regsOverlap(Reg, AntiHint);
- });
- };
-
HintsAndCustomOrder.truncate(NumHints);
HintsAndCustomOrder.append(Order.begin(), Order.end());
- // Partition non-anti-hinted register go first
+ // Custom reordering of the allocation order.
+ filterAndSortForAntiHintedRegs(
+ VirtReg,
+ MutableArrayRef<MCPhysReg>(HintsAndCustomOrder).drop_front(NumHints),
+ AntiHintedRegUnits, MF, Matrix);
+}
+
+void TargetRegisterInfo::filterAndSortForAntiHintedRegs(
+ Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
+ const LiveRegMatrix *Matrix) const {
+
+ // Partition non-anti-hinted register go first.
auto *PartitionPoint = std::stable_partition(
- HintsAndCustomOrder.begin() + NumHints, HintsAndCustomOrder.end(),
- [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
+ CustomOrder.begin(), CustomOrder.end(),
+ [&](MCPhysReg Reg) { return !isAntiHintedReg(Reg, AntiHintedRegUnits); });
LLVM_DEBUG({
size_t NonAntiHintedCount =
- std::distance(HintsAndCustomOrder.begin() + NumHints, PartitionPoint);
- size_t AntiHintedCount =
- std::distance(PartitionPoint, HintsAndCustomOrder.end());
+ std::distance(CustomOrder.begin(), PartitionPoint);
+ size_t AntiHintedCount = std::distance(PartitionPoint, CustomOrder.end());
dbgs() << "Added " << NonAntiHintedCount
<< " non-anti-hinted registers first\n"
<< "Added " << AntiHintedCount
diff --git a/llvm/unittests/CodeGen/AllocationOrderTest.cpp b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
index aa636b6673b1e..5346d1b1863aa 100644
--- a/llvm/unittests/CodeGen/AllocationOrderTest.cpp
+++ b/llvm/unittests/CodeGen/AllocationOrderTest.cpp
@@ -117,7 +117,7 @@ TEST(AllocationOrderTest, IsHintTest) {
EXPECT_EQ(V, 5U);
}
-TEST(AllocationOrderTest, AntiHintShuffleOrderOverridesOrder) {
+TEST(AllocationOrderTest, AntiHintCustomOrderOverridesOrder) {
SmallVector<MCPhysReg, 16> HintsAndCustomOrder = {1, 2, 5, 3, 4};
SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
AllocationOrder O(std::move(HintsAndCustomOrder), 2, Order, false);
@@ -131,7 +131,7 @@ TEST(AllocationOrderTest, EmptyAntiHintedCustomOrderFallsBackToOrder) {
EXPECT_EQ((std::vector<MCPhysReg>{1, 2, 3, 4, 5}), loadOrder(O));
}
-TEST(AllocationOrderTest, HardHintsIgnoreAntiHintShuffleOrder) {
+TEST(AllocationOrderTest, HardHintsIgnoreAntiHintCustomOrder) {
SmallVector<MCPhysReg, 16> HintsAndCustomOrder = {1, 2, 5, 3, 4};
SmallVector<MCPhysReg, 16> Order = {3, 4, 5};
AllocationOrder O(std::move(HintsAndCustomOrder), 2, Order, true);
>From f81b77befe88133209d38a941182a0ec58678b01 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Fri, 4 Sep 2026 22:52:30 -0500
Subject: [PATCH 12/22] Addressed review and added target overrideable
filterAndSort
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 36 +++++++----------------
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 8 ++---
2 files changed, 13 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 93c9b1f8e948c..a6124cf8fc89c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4225,15 +4225,10 @@ bool SIRegisterInfo::shouldApplyAntiHints(
return true;
}
-void SIRegisterInfo::applyRegAllocationAntiHints(
- Register VirtReg, ArrayRef<MCPhysReg> Order,
- SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
- SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
- const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
-
- // Early exit to default order if we have no anti-hints or no VRM.
- if (AntiHints.empty() || !VRM)
- return;
+void SIRegisterInfo::filterAndSortForAntiHintedRegs(
+ Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
+ const LiveRegMatrix *Matrix) const {
// Get total number of allocated VGPRs to determine the current occupancy.
unsigned NumVGPRs = 0;
@@ -4278,32 +4273,21 @@ void SIRegisterInfo::applyRegAllocationAntiHints(
MaxVGPRsForCurrentOccupancy;
};
- HintsAndCustomOrder.truncate(NumHints);
- HintsAndCustomOrder.append(Order.begin(), Order.end());
-
- // Helper to check if a register overlaps with any anti-hint.
- auto isAntiHinted = [&](MCPhysReg Reg) {
- return llvm::any_of(AntiHints, [&](MCPhysReg AntiHint) {
- return regsOverlap(Reg, AntiHint);
- });
- };
-
// Find the cutoff point for the current occupancy VGPR budget.
- auto *BeyondBudgetStart =
- llvm::find_if(llvm::drop_begin(HintsAndCustomOrder, NumHints),
- [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
+ auto *BeyondBudgetStart = llvm::find_if(
+ CustomOrder, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
// Only shuffle within the current occupancy VGPR budget.
auto *PartitionPoint = std::stable_partition(
- HintsAndCustomOrder.begin() + NumHints, BeyondBudgetStart,
- [&](MCPhysReg Reg) { return !isAntiHinted(Reg); });
+ CustomOrder.begin(), BeyondBudgetStart,
+ [&](MCPhysReg Reg) { return !isAntiHintedReg(Reg, AntiHintedRegUnits); });
LLVM_DEBUG({
size_t NonAntiHintedCount =
- std::distance(HintsAndCustomOrder.begin() + NumHints, PartitionPoint);
+ std::distance(CustomOrder.begin(), PartitionPoint);
size_t AntiHintedCount = std::distance(PartitionPoint, BeyondBudgetStart);
size_t BeyondBudgetCount =
- std::distance(BeyondBudgetStart, HintsAndCustomOrder.end());
+ std::distance(BeyondBudgetStart, CustomOrder.end());
dbgs() << "Added " << NonAntiHintedCount
<< " non-anti-hinted registers first\n"
<< "Added " << AntiHintedCount
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 7765849ab804b..a704767e3b749 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -373,11 +373,9 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
unsigned NumAllocatedVGPRs,
unsigned &MaxVGPRsForCurrentOccupancy) const;
- void applyRegAllocationAntiHints(
- Register VirtReg, ArrayRef<MCPhysReg> Order,
- SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
- SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
- const VirtRegMap *VRM = nullptr,
+ void filterAndSortForAntiHintedRegs(
+ Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
const LiveRegMatrix *Matrix = nullptr) const override;
const int *getRegUnitPressureSets(MCRegUnit RegUnit) const override;
>From 99f6c74ba2d0ad33772bd95d98393670f46f5a42 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sat, 5 Sep 2026 10:52:01 -0500
Subject: [PATCH 13/22] Addressed review
---
llvm/include/llvm/CodeGen/MIRYamlMapping.h | 3 ++-
llvm/lib/CodeGen/MIRParser/MIRParser.cpp | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 142ace2cf765f..27c9cc8301cba 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -232,9 +232,10 @@ template <> struct MappingTraits<VirtualRegisterDefinition> {
// so a plain mapOptional with an empty default would still emit the keys
// and change every existing test's output.
// Skip the call on output when empty to keep them off entirely.
- if (!YamlIO.outputting() || !Reg.AntiHints.empty())
+ if (!YamlIO.outputting() || !Reg.AntiHints.empty()) {
YamlIO.mapOptional("anti-hints", Reg.AntiHints,
std::vector<FlowStringValue>());
+ }
if (!YamlIO.outputting() || !Reg.SplitFrom.Value.empty())
YamlIO.mapOptional("split-from", Reg.SplitFrom, StringValue());
if (!YamlIO.outputting() || !Reg.AssignedPhys.Value.empty())
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index febb010bb4d09..965e7286d8cef 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -774,7 +774,7 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS,
}
if (!VReg.AntiHints.empty() && Info.Kind != VRegInfo::NORMAL)
return error(VReg.AntiHints.front().SourceRange.Start,
- Twine("anti-hints can only be set for normal vregs"));
+ "anti-hints can only be set for normal vregs");
for (const auto &AntiHintValue : VReg.AntiHints) {
Register AntiHintReg;
>From 22cdb36c98424b4bde30e6ff70ab9907ea0f035a Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sat, 12 Sep 2026 13:34:40 -0500
Subject: [PATCH 14/22] Addressed review
---
llvm/include/llvm/CodeGen/TargetRegisterInfo.h | 7 +++++--
llvm/lib/CodeGen/AllocationOrder.cpp | 3 ++-
llvm/lib/CodeGen/AllocationOrder.h | 5 +++++
llvm/lib/CodeGen/RegAllocGreedy.cpp | 5 ++++-
llvm/lib/CodeGen/TargetRegisterInfo.cpp | 6 +++---
5 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 9de9e644effda..182a8984bae4d 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -39,6 +39,7 @@ class DIExpression;
class LiveRegMatrix;
class MachineFunction;
class MachineInstr;
+class RegisterClassInfo;
class RegScavenger;
class VirtRegMap;
class LiveIntervals;
@@ -861,13 +862,15 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
Register VirtReg, ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix = nullptr) const;
+ const LiveRegMatrix *Matrix = nullptr,
+ const RegisterClassInfo *RegClassInfo = nullptr) const;
/// Custom reordering of the allocation order.
virtual void filterAndSortForAntiHintedRegs(
Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix = nullptr) const;
+ const LiveRegMatrix *Matrix = nullptr,
+ const RegisterClassInfo *RegClassInfo = nullptr) const;
/// Allow the target to reverse allocation order of local live ranges. This
/// will generally allocate shorter local live ranges first. For targets with
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index ac4485c849a00..705fcef23030b 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -74,7 +74,8 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
if (AntiHintedRegUnits.any()) {
HintsAndCustomOrder.reserve(NumHints + Order.size());
TRI->applyRegAllocationAntiHints(VirtReg, Order, HintsAndCustomOrder,
- NumHints, AntiHintedRegUnits, MF, Matrix);
+ NumHints, AntiHintedRegUnits, MF, Matrix,
+ &RegClassInfo);
}
// Create allocation order object.
diff --git a/llvm/lib/CodeGen/AllocationOrder.h b/llvm/lib/CodeGen/AllocationOrder.h
index 09d8fd68de17b..7c61c01d9ea59 100644
--- a/llvm/lib/CodeGen/AllocationOrder.h
+++ b/llvm/lib/CodeGen/AllocationOrder.h
@@ -129,6 +129,11 @@ class LLVM_LIBRARY_VISIBILITY AllocationOrder {
/// Get the allocation order without reordered hints.
ArrayRef<MCPhysReg> getOrder() const { return Order; }
+ /// Return true if a custom order replaced the RegisterClassInfo order.
+ bool hasCustomOrder() const {
+ return static_cast<int>(HintsAndCustomOrder.size()) > NumHints;
+ }
+
/// Return true if Reg is a preferred physical register.
bool isHint(Register Reg) const {
assert(!Reg.isPhysical() ||
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index fc28b229dcddc..42de9fc7f6f75 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -683,7 +683,10 @@ RegAllocEvictionAdvisor::getOrderLimit(const LiveInterval &VirtReg,
// It is normal for register classes to have a long tail of registers with
// the same cost. We don't need to look at them if they're too expensive.
- if (RegCosts[Order.getOrder().back()] >= CostPerUseLimit) {
+ // LastCostChange is an index into the original RegisterClassInfo order, so
+ // it cannot be used to shorten a custom order.
+ if (!Order.hasCustomOrder() &&
+ RegCosts[Order.getOrder().back()] >= CostPerUseLimit) {
OrderLimit = RegClassInfo.getLastCostChange(RC);
LLVM_DEBUG(dbgs() << "Only trying the first " << OrderLimit
<< " regs.\n");
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index f6ec5c9ac46e0..7db9732213ed5 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -454,7 +454,7 @@ void TargetRegisterInfo::applyRegAllocationAntiHints(
Register VirtReg, ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix) const {
+ const LiveRegMatrix *Matrix, const RegisterClassInfo *RegClassInfo) const {
if (AntiHintedRegUnits.none())
return;
@@ -466,13 +466,13 @@ void TargetRegisterInfo::applyRegAllocationAntiHints(
filterAndSortForAntiHintedRegs(
VirtReg,
MutableArrayRef<MCPhysReg>(HintsAndCustomOrder).drop_front(NumHints),
- AntiHintedRegUnits, MF, Matrix);
+ AntiHintedRegUnits, MF, Matrix, RegClassInfo);
}
void TargetRegisterInfo::filterAndSortForAntiHintedRegs(
Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix) const {
+ const LiveRegMatrix *Matrix, const RegisterClassInfo *RegClassInfo) const {
// Partition non-anti-hinted register go first.
auto *PartitionPoint = std::stable_partition(
>From 6ecc29f704356eec9bcc954f0b401983aa802f70 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sat, 12 Sep 2026 13:41:13 -0500
Subject: [PATCH 15/22] Addressed review
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 85 ++++++----
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 8 +-
.../AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 156 ++++++++++++++++++
3 files changed, 212 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index a6124cf8fc89c..598e836280963 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4201,8 +4201,8 @@ bool SIRegisterInfo::shouldApplyAntiHints(
// Do not apply anti-hints if we are reaching close to the VGPR budget. For
// target occupancy, the 80% cutoff is a conservative: anti-hints are disabled
// early enough that later registers still have headroom to stay at target
- // occupancy. For current occupancy, the 95% cutoff margin is used to apply
- // anti-hints close to the limit of the current occupancy budget.
+ // occupancy. For current occupancy, the 95% cutoff margin is used to not
+ // apply anti-hints close to the limit of the current occupancy budget.
unsigned MaxVGPRsCutOffForTargetOccupancy =
(ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100;
unsigned MaxVGPRsCutOffForCurrentOccupancy =
@@ -4225,22 +4225,56 @@ bool SIRegisterInfo::shouldApplyAntiHints(
return true;
}
+// Returns true if Reg fits within the current occupancy VGPR budget.
+bool SIRegisterInfo::isRegWithinOccupancyBudget(
+ MCPhysReg Reg, unsigned NumVGPRs, unsigned NumAGPRs,
+ unsigned MaxVGPRsForCurrentOccupancy) const {
+ const TargetRegisterClass *RC = getPhysRegBaseClass(Reg);
+
+ // No VGPR or AGPR usage.
+ if (!RC || (!isVGPRClass(RC) && !isAGPRClass(RC)))
+ return true;
+
+ unsigned RegEndIndex =
+ getHWRegIndex(Reg) + divideCeil(getRegSizeInBits(*RC), 32);
+
+ unsigned MaxVGPR = NumVGPRs;
+ unsigned MaxAGPR = NumAGPRs;
+
+ if (isAGPRClass(RC))
+ MaxAGPR = std::max(MaxAGPR, RegEndIndex);
+ else
+ MaxVGPR = std::max(MaxVGPR, RegEndIndex);
+
+ return static_cast<unsigned>(
+ AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), MaxAGPR, MaxVGPR)) <=
+ MaxVGPRsForCurrentOccupancy;
+}
+
void SIRegisterInfo::filterAndSortForAntiHintedRegs(
Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix) const {
+ const LiveRegMatrix *Matrix, const RegisterClassInfo *RegClassInfo) const {
+
+ if (none_of(CustomOrder, [&](MCPhysReg Reg) {
+ return isAntiHintedReg(Reg, AntiHintedRegUnits);
+ }))
+ return;
- // Get total number of allocated VGPRs to determine the current occupancy.
- unsigned NumVGPRs = 0;
- unsigned NumAGPRs = 0;
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ unsigned NumVGPRs = getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
+ /*IncludeCalls=*/false);
+ unsigned NumAGPRs = getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
+ /*IncludeCalls=*/false);
if (Matrix) {
- for (MCPhysReg Reg : AMDGPU::VGPR_32RegClass)
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::VGPR_32RegClass))
if (Matrix->isPhysRegUsed(Reg))
NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
- for (MCPhysReg Reg : AMDGPU::AGPR_32RegClass)
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::AGPR_32RegClass))
if (Matrix->isPhysRegUsed(Reg))
NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
}
+
unsigned NumAllocatedVGPRs =
AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs);
@@ -4249,35 +4283,14 @@ void SIRegisterInfo::filterAndSortForAntiHintedRegs(
if (!shouldApplyAntiHints(MF, NumAllocatedVGPRs, MaxVGPRsForCurrentOccupancy))
return;
- // Returns true if Reg fits within the current occupancy VGPR budget.
- auto IsWithinBudget = [&](MCPhysReg Reg) -> bool {
- const TargetRegisterClass *RC = getPhysRegBaseClass(Reg);
-
- // No VGPR or AGPR usage.
- if (!RC ||
- (!isVGPRClass(RC) && !isAGPRClass(RC) && !isVectorSuperClass(RC)))
- return true;
-
- unsigned RegEndIndex =
- getHWRegIndex(Reg) + divideCeil(getRegSizeInBits(*RC), 32);
-
- unsigned MaxVGPR = NumVGPRs;
- unsigned MaxAGPR = NumAGPRs;
- if (isAGPRClass(RC))
- MaxAGPR = std::max(MaxAGPR, RegEndIndex);
- else
- MaxVGPR = std::max(MaxVGPR, RegEndIndex);
-
- return static_cast<unsigned>(AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(),
- MaxAGPR, MaxVGPR)) <=
- MaxVGPRsForCurrentOccupancy;
- };
-
- // Find the cutoff point for the current occupancy VGPR budget.
- auto *BeyondBudgetStart = llvm::find_if(
- CustomOrder, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); });
+ // Reorder all in-budget first so the anti-hinted partition covers
+ // both VGPRs and AGPRs.
+ auto *BeyondBudgetStart = std::stable_partition(
+ CustomOrder.begin(), CustomOrder.end(), [&](MCPhysReg Reg) {
+ return isRegWithinOccupancyBudget(Reg, NumVGPRs, NumAGPRs,
+ MaxVGPRsForCurrentOccupancy);
+ });
- // Only shuffle within the current occupancy VGPR budget.
auto *PartitionPoint = std::stable_partition(
CustomOrder.begin(), BeyondBudgetStart,
[&](MCPhysReg Reg) { return !isAntiHintedReg(Reg, AntiHintedRegUnits); });
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index a704767e3b749..331dc858c1518 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -63,6 +63,11 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
void reserveRegisterTuples(BitVector &, MCRegister Reg) const;
+ /// True if assigning Reg would fit in the current occupancy VGPR budget.
+ bool isRegWithinOccupancyBudget(MCPhysReg Reg, unsigned NumVGPRs,
+ unsigned NumAGPRs,
+ unsigned MaxVGPRsForCurrentOccupancy) const;
+
public:
SIRegisterInfo(const GCNSubtarget &ST);
@@ -376,7 +381,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
void filterAndSortForAntiHintedRegs(
Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix = nullptr) const override;
+ const LiveRegMatrix *Matrix = nullptr,
+ const RegisterClassInfo *RegClassInfo = nullptr) const override;
const int *getRegUnitPressureSets(MCRegUnit RegUnit) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
index 2eec9fe8c8eb2..8224ed73fcf1d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
@@ -9,8 +9,11 @@
define amdgpu_kernel void @anti_hints_suppressed_by_current_budget() #0 { ret void }
define amdgpu_kernel void @anti_hints_unified_agpr_headroom () #1 { ret void }
define amdgpu_kernel void @anti_hints_unified_agpr_budget () #1 { ret void }
+ define amdgpu_kernel void @anti_hints_high_vgpr_counts_for_budget() #0 { ret void }
+ define amdgpu_kernel void @anti_hints_av_class_applied_to_agpr() #2 { ret void }
attributes #0 = { nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
attributes #1 = { nounwind "frame-pointer"="none"}
+ attributes #2 = { nounwind "frame-pointer"="none" "amdgpu-num-vgpr"="20" "amdgpu-agpr-alloc"="36"}
...
---
@@ -578,3 +581,156 @@ body: |
%8.sub12_sub13_sub14_sub15:vreg_512_align2 = DS_READ_B128_gfx9 %0.sub0, 6144, 0, implicit $exec :: (load (s128), addrspace 3)
S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %5, implicit %6, implicit %7, implicit %8
...
+
+---
+name: anti_hints_high_vgpr_counts_for_budget
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 8
+registers:
+ - { id: 0, class: vreg_128_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sgpr_128 }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: vgpr_32 }
+ - { id: 6, class: vreg_128_align2 }
+ - { id: 7, class: vreg_512_align2 }
+ - { id: 8, class: vreg_512_align2 }
+ - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] }
+ - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] }
+ - { id: 13, class: vreg_512_align2 }
+ - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] }
+ - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] }
+ - { id: 17, class: vreg_512_align2 }
+ - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] }
+ - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] }
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4' }
+ - { reg: '$vgpr5' }
+ - { reg: '$vgpr52_vgpr53_vgpr54_vgpr55' }
+ - { reg: '$vgpr56_vgpr57_vgpr58_vgpr59' }
+ - { reg: '$vgpr60_vgpr61_vgpr62_vgpr63' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-LABEL: name: anti_hints_high_vgpr_counts_for_budget
+ ; GFX942: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX942-NEXT: S_NOP 4
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX942-NEXT: S_NOP 5
+ ; GFX942-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: S_NOP 6
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX942-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX942-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit $vgpr52_vgpr53_vgpr54_vgpr55, implicit $vgpr56_vgpr57_vgpr58_vgpr59, implicit $vgpr60_vgpr61_vgpr62_vgpr63
+ ;
+ ; GFX90A-LABEL: name: anti_hints_high_vgpr_counts_for_budget
+ ; GFX90A: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec
+ ; GFX90A-NEXT: S_NOP 12
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec
+ ; GFX90A-NEXT: S_NOP 13
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: S_NOP 14
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GFX90A-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit $vgpr52_vgpr53_vgpr54_vgpr55, implicit $vgpr56_vgpr57_vgpr58_vgpr59, implicit $vgpr60_vgpr61_vgpr62_vgpr63
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = COPY $vgpr4
+ %2:vgpr_32 = COPY $vgpr5
+ %3:sgpr_128 = IMPLICIT_DEF
+ %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec
+ %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec
+ %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ %7:vreg_512_align2 = IMPLICIT_DEF
+ %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec
+ %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3)
+ %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec
+ %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec
+ %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec
+ %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3)
+ %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec
+ %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec
+ %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3)
+ %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit $vgpr52_vgpr53_vgpr54_vgpr55, implicit $vgpr56_vgpr57_vgpr58_vgpr59, implicit $vgpr60_vgpr61_vgpr62_vgpr63
+...
+
+---
+name: anti_hints_av_class_applied_to_agpr
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 6
+registers:
+ - { id: 0, class: vreg_64_align2 }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: areg_512_align2 }
+ - { id: 3, class: areg_512_align2 }
+ - { id: 4, class: av_128_align2, anti-hints: [ '%2' ] }
+liveins:
+ - { reg: '$vgpr0_vgpr1' }
+ - { reg: '$vgpr2' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1, $vgpr2
+ ; AH-LABEL: name: anti_hints_av_class_applied_to_agpr
+ ; AH: liveins: $vgpr2, $vgpr0_vgpr1
+ ; AH-NEXT: {{ $}}
+ ; AH-NEXT: renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+ ; AH-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = contract V_MFMA_F32_32X32X8F16_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+ ; AH-NEXT: renamable $agpr32_agpr33_agpr34_agpr35 = DS_READ_B128_gfx9 renamable $vgpr2, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ ; AH-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr2, killed renamable $agpr32_agpr33_agpr34_agpr35, 2048, 0, implicit $exec :: (store (s128), addrspace 3)
+ ; AH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+ %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+ %1:vgpr_32 = COPY $vgpr2
+ %2:areg_512_align2 = IMPLICIT_DEF
+ early-clobber %3:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_e64 %0, %0, killed %2, 0, 0, 0, implicit $mode, implicit $exec
+ %4:av_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+ DS_WRITE_B128_gfx9 %1, %4, 2048, 0, implicit $exec :: (store (s128), addrspace 3)
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %3
+...
>From 6dc04cf3a99d4adbf2745a40c4d9f30a59907a81 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Tue, 15 Sep 2026 19:30:16 -0500
Subject: [PATCH 16/22] addressed review
---
llvm/include/llvm/CodeGen/MachineRegisterInfo.h | 4 ++--
llvm/include/llvm/CodeGen/TargetRegisterInfo.h | 14 +++++++-------
llvm/lib/CodeGen/AllocationOrder.cpp | 4 +---
llvm/lib/CodeGen/TargetRegisterInfo.cpp | 5 +++--
4 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 69d14fd11507f..8a50624d4fb27 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -917,7 +917,7 @@ class MachineRegisterInfo {
/// Add a register allocation anti-hint for the specified virtual register.
/// This tells the allocator to avoid allocating VReg to the same physical
/// register as AntiHintVReg (or overlapping ones).
- void addRegAllocAntiHint(Register VReg, Register AntiHintVReg) {
+ void addRegAllocationAntiHint(Register VReg, Register AntiHintVReg) {
assert(VReg.isVirtual() && AntiHintVReg.isVirtual() &&
"Anti-hints and anti-hint targets are only for virtual registers");
AntiHintRegs.grow(VReg);
@@ -931,7 +931,7 @@ class MachineRegisterInfo {
void addRegAllocationAntiHints(Register VReg,
ArrayRef<Register> AntiHintVRegs) {
for (Register AntiHint : AntiHintVRegs)
- addRegAllocAntiHint(VReg, AntiHint);
+ addRegAllocationAntiHint(VReg, AntiHint);
}
/// Clear all anti-hints for a register.
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 182a8984bae4d..449013cab7970 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -703,6 +703,13 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return RCInfos[getNumRegClasses() * HwMode + RC.getID()];
}
+ /// Custom reordering of the allocation order.
+ virtual void filterAndSortForAntiHintedRegs(
+ Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
+ const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
+ const LiveRegMatrix *Matrix = nullptr,
+ const RegisterClassInfo *RegClassInfo = nullptr) const;
+
public:
/// Returns the register class associated with the enumeration value.
/// See class MCOperandInfo.
@@ -865,13 +872,6 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
const LiveRegMatrix *Matrix = nullptr,
const RegisterClassInfo *RegClassInfo = nullptr) const;
- /// Custom reordering of the allocation order.
- virtual void filterAndSortForAntiHintedRegs(
- Register VirtReg, MutableArrayRef<MCPhysReg> CustomOrder,
- const BitVector &AntiHintedRegUnits, const MachineFunction &MF,
- const LiveRegMatrix *Matrix = nullptr,
- const RegisterClassInfo *RegClassInfo = nullptr) const;
-
/// Allow the target to reverse allocation order of local live ranges. This
/// will generally allocate shorter local live ranges first. For targets with
/// many registers, this could reduce regalloc compile time by a large
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index 705fcef23030b..55be86dab2b22 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -71,12 +71,10 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
}
});
- if (AntiHintedRegUnits.any()) {
- HintsAndCustomOrder.reserve(NumHints + Order.size());
+ if (AntiHintedRegUnits.any())
TRI->applyRegAllocationAntiHints(VirtReg, Order, HintsAndCustomOrder,
NumHints, AntiHintedRegUnits, MF, Matrix,
&RegClassInfo);
- }
// Create allocation order object.
AllocationOrder AO(std::move(HintsAndCustomOrder), NumHints, Order,
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index 7db9732213ed5..79b01d27d91a5 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -459,7 +459,8 @@ void TargetRegisterInfo::applyRegAllocationAntiHints(
if (AntiHintedRegUnits.none())
return;
- HintsAndCustomOrder.truncate(NumHints);
+ assert(HintsAndCustomOrder.size() == NumHints &&
+ "HintsAndCustomOrder should only contain the hints here.");
HintsAndCustomOrder.append(Order.begin(), Order.end());
// Custom reordering of the allocation order.
@@ -475,7 +476,7 @@ void TargetRegisterInfo::filterAndSortForAntiHintedRegs(
const LiveRegMatrix *Matrix, const RegisterClassInfo *RegClassInfo) const {
// Partition non-anti-hinted register go first.
- auto *PartitionPoint = std::stable_partition(
+ [[maybe_unused]] auto *PartitionPoint = std::stable_partition(
CustomOrder.begin(), CustomOrder.end(),
[&](MCPhysReg Reg) { return !isAntiHintedReg(Reg, AntiHintedRegUnits); });
>From e339233fb03da60fb0df93a2045b0c92ca5ecff6 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Tue, 15 Sep 2026 19:33:57 -0500
Subject: [PATCH 17/22] addressed review
---
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
index 8224ed73fcf1d..55c856f7a33d6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir
@@ -13,7 +13,7 @@
define amdgpu_kernel void @anti_hints_av_class_applied_to_agpr() #2 { ret void }
attributes #0 = { nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
attributes #1 = { nounwind "frame-pointer"="none"}
- attributes #2 = { nounwind "frame-pointer"="none" "amdgpu-num-vgpr"="20" "amdgpu-agpr-alloc"="36"}
+ attributes #2 = { nounwind "frame-pointer"="none" "amdgpu-waves-per-eu"="8,8" "amdgpu-agpr-alloc"="60"}
...
---
>From 099e49a72ce331b98d3ade91fb7d0bacd43db5b3 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 16 Sep 2026 11:52:53 -0500
Subject: [PATCH 18/22] Addressed reviews
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 57 ++++++++++++-----------
1 file changed, 31 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 598e836280963..44e893f5ed7e0 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4183,15 +4183,16 @@ bool SIRegisterInfo::shouldApplyAntiHints(
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
- unsigned TargetOccupancy = MFI->getOccupancy();
+ unsigned RecordedMaxOccupancy = MFI->getOccupancy();
unsigned CurrentOccupancy =
ST.getOccupancyWithNumVGPRs(NumAllocatedVGPRs, DynamicVGPRBlockSize);
MaxVGPRsForCurrentOccupancy =
ST.getMaxNumVGPRs(CurrentOccupancy, DynamicVGPRBlockSize);
LLVM_DEBUG(dbgs() << "anti-hints: " << NumAllocatedVGPRs
- << " VGPRs allocated, target occupancy " << TargetOccupancy
- << ", current occupancy " << CurrentOccupancy << '\n');
+ << " VGPRs allocated, RecordedMaxOccupancy "
+ << RecordedMaxOccupancy << ", current occupancy "
+ << CurrentOccupancy << '\n');
// If we are already at lowest occupancy, then there is no need to protect
// against occupancy regression.
@@ -4199,19 +4200,21 @@ bool SIRegisterInfo::shouldApplyAntiHints(
return true;
// Do not apply anti-hints if we are reaching close to the VGPR budget. For
- // target occupancy, the 80% cutoff is a conservative: anti-hints are disabled
- // early enough that later registers still have headroom to stay at target
- // occupancy. For current occupancy, the 95% cutoff margin is used to not
- // apply anti-hints close to the limit of the current occupancy budget.
- unsigned MaxVGPRsCutOffForTargetOccupancy =
- (ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100;
+ // recorded max occupancy, the 80% cutoff is a conservative: anti-hints are
+ // disabled early enough that later registers still have headroom to stay at
+ // recorded max occupancy. For current occupancy, the 95% cutoff margin is
+ // used to not apply anti-hints close to the limit of the current occupancy
+ // budget.
+ unsigned MaxVGPRsCutOffForRecordedMaxOccupancy =
+ (ST.getMaxNumVGPRs(RecordedMaxOccupancy, DynamicVGPRBlockSize) * 80) /
+ 100;
unsigned MaxVGPRsCutOffForCurrentOccupancy =
(MaxVGPRsForCurrentOccupancy * 95) / 100;
- if (NumAllocatedVGPRs >= MaxVGPRsCutOffForTargetOccupancy) {
+ if (NumAllocatedVGPRs >= MaxVGPRsCutOffForRecordedMaxOccupancy) {
LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the "
- << MaxVGPRsCutOffForTargetOccupancy
- << " VGPR cutoff for target occupancy\n");
+ << MaxVGPRsCutOffForRecordedMaxOccupancy
+ << " VGPR cutoff for RecordedMaxOccupancy\n");
return false;
}
@@ -4232,7 +4235,7 @@ bool SIRegisterInfo::isRegWithinOccupancyBudget(
const TargetRegisterClass *RC = getPhysRegBaseClass(Reg);
// No VGPR or AGPR usage.
- if (!RC || (!isVGPRClass(RC) && !isAGPRClass(RC)))
+ if (!RC || !hasVectorRegisters(RC))
return true;
unsigned RegEndIndex =
@@ -4262,18 +4265,20 @@ void SIRegisterInfo::filterAndSortForAntiHintedRegs(
return;
const MachineRegisterInfo &MRI = MF.getRegInfo();
- unsigned NumVGPRs = getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
- /*IncludeCalls=*/false);
- unsigned NumAGPRs = getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
- /*IncludeCalls=*/false);
- if (Matrix) {
- for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::VGPR_32RegClass))
- if (Matrix->isPhysRegUsed(Reg))
- NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
- for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::AGPR_32RegClass))
- if (Matrix->isPhysRegUsed(Reg))
- NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
- }
+ assert(hasVectorRegisters(MRI.getRegClass(VirtReg)) &&
+ "SGPR anti-hints are not handled");
+ unsigned NumVGPRs = 0;
+ unsigned NumAGPRs = 0;
+
+ assert(RegClassInfo && "RegClassInfo required to compute occupancy");
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::VGPR_32RegClass))
+ if ((Matrix && Matrix->isPhysRegUsed(Reg)) ||
+ MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
+ NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::AGPR_32RegClass))
+ if ((Matrix && Matrix->isPhysRegUsed(Reg)) ||
+ MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
+ NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
unsigned NumAllocatedVGPRs =
AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs);
@@ -4291,7 +4296,7 @@ void SIRegisterInfo::filterAndSortForAntiHintedRegs(
MaxVGPRsForCurrentOccupancy);
});
- auto *PartitionPoint = std::stable_partition(
+ [[maybe_unused]] auto *PartitionPoint = std::stable_partition(
CustomOrder.begin(), BeyondBudgetStart,
[&](MCPhysReg Reg) { return !isAntiHintedReg(Reg, AntiHintedRegUnits); });
>From c409adea52ad8ee5b121860df91c6e4c24b4648a Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Wed, 16 Sep 2026 22:15:27 -0500
Subject: [PATCH 19/22] Addressed reveiw
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 44e893f5ed7e0..5cccfd0e3d9be 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4189,10 +4189,9 @@ bool SIRegisterInfo::shouldApplyAntiHints(
MaxVGPRsForCurrentOccupancy =
ST.getMaxNumVGPRs(CurrentOccupancy, DynamicVGPRBlockSize);
- LLVM_DEBUG(dbgs() << "anti-hints: " << NumAllocatedVGPRs
- << " VGPRs allocated, RecordedMaxOccupancy "
- << RecordedMaxOccupancy << ", current occupancy "
- << CurrentOccupancy << '\n');
+ LLVM_DEBUG(dbgs() << "anti-hints: VGPRs allocated = " << NumAllocatedVGPRs
+ << ", RecordedMaxOccupancy = " << RecordedMaxOccupancy
+ << ", current occupancy = " << CurrentOccupancy << '\n');
// If we are already at lowest occupancy, then there is no need to protect
// against occupancy regression.
@@ -4270,15 +4269,19 @@ void SIRegisterInfo::filterAndSortForAntiHintedRegs(
unsigned NumVGPRs = 0;
unsigned NumAGPRs = 0;
+ assert(Matrix && "LiveRegMatrix required to compute occupancy");
assert(RegClassInfo && "RegClassInfo required to compute occupancy");
- for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::VGPR_32RegClass))
- if ((Matrix && Matrix->isPhysRegUsed(Reg)) ||
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::VGPR_32RegClass)) {
+ if (Matrix->isPhysRegUsed(Reg) ||
MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
- for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::AGPR_32RegClass))
- if ((Matrix && Matrix->isPhysRegUsed(Reg)) ||
+ }
+
+ for (MCPhysReg Reg : RegClassInfo->getOrder(&AMDGPU::AGPR_32RegClass)) {
+ if (Matrix->isPhysRegUsed(Reg) ||
MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
+ }
unsigned NumAllocatedVGPRs =
AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs);
>From 6d1b9d6cc8202c990a75f30e80f2e4d3c220566e Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Thu, 24 Sep 2026 14:16:57 -0500
Subject: [PATCH 20/22] Addressed review and simplification
---
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp | 46 ++++++++++---------
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 10 ++--
.../CodeGen/AMDGPU/anti-hints-mfma-war.mir | 12 ++---
.../CodeGen/AMDGPU/anti-hints-mfma-waw.mir | 12 ++---
.../llvm.amdgcn.mfma.anti-hints.gfx942.mir | 18 ++------
5 files changed, 46 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
index 231b1adaea871..07d565cef3fa4 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
@@ -27,10 +27,23 @@ using namespace llvm::AMDGPU;
namespace HC = llvm::AMDGPU::HazardClass;
-static cl::opt<std::string> AntiHintRuleSelection(
- "amdgpu-anti-hints-rules", cl::Hidden,
- cl::desc("Comma-separated anti-hints rules (waw, war), or all or none."),
- cl::init("all"));
+enum class AntiHintRule {
+ None,
+ MFMAWAW,
+ MFMAWAR,
+ All,
+};
+
+static cl::bits<AntiHintRule> AntiHintRuleSelection(
+ "amdgpu-anti-hints-rules", cl::Hidden, cl::CommaSeparated,
+ cl::desc("Anti-hints rules to select."),
+ cl::values(clEnumValN(AntiHintRule::None, "none", "Select no rules"),
+ clEnumValN(AntiHintRule::MFMAWAW, "mfma-waw",
+ "MFMA destination write-after-write"),
+ clEnumValN(AntiHintRule::MFMAWAR, "mfma-war",
+ "XDL MFMA src2 write-after-read"),
+ clEnumValN(AntiHintRule::All, "all",
+ "Select all rules (default)")));
namespace {
@@ -203,25 +216,19 @@ bool hasMFMAHazard(const HazardContext &Ctx) {
return Ctx.ST->hasGFX90AInsts();
}
-bool ruleSelected(StringRef Name) {
- StringRef Selection(AntiHintRuleSelection);
- if (Selection.equals_insensitive("all"))
+bool ruleSelected(AntiHintRule Rule) {
+ if (!AntiHintRuleSelection.getNumOccurrences() ||
+ AntiHintRuleSelection.isSet(AntiHintRule::All))
return true;
- if (Selection.equals_insensitive("none"))
- return false;
- SmallVector<StringRef, 3> Selected;
- Selection.split(Selected, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
- return llvm::any_of(Selected, [Name](StringRef S) {
- return S.trim().equals_insensitive(Name);
- });
+ return AntiHintRuleSelection.isSet(Rule);
}
bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
- return hasMFMAHazard(Ctx) && ruleSelected("waw");
+ return hasMFMAHazard(Ctx) && ruleSelected(AntiHintRule::MFMAWAW);
}
bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
- return hasMFMAHazard(Ctx) && ruleSelected("war");
+ return hasMFMAHazard(Ctx) && ruleSelected(AntiHintRule::MFMAWAR);
}
bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
@@ -477,16 +484,13 @@ class AntiHintEngine {
if (!Ctx.LIS->hasInterval(ProducerReg))
return;
const LiveInterval &ProducerLI = Ctx.LIS->getInterval(ProducerReg);
+ // Skip a live producer reg.
if (ProducerLI.liveAt(Slot))
return;
for (Register ConsumerReg : ConsumerRegs) {
if (ConsumerReg == ProducerReg || isCopyOf(ConsumerReg, ProducerReg))
continue;
- // Only allow anti-hints when producer and consumer reg live ranges do
- // not overlap
- if (Ctx.LIS->hasInterval(ConsumerReg) &&
- ProducerLI.overlaps(Ctx.LIS->getInterval(ConsumerReg)))
- continue;
+
Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
if (CT.Hint == ConsumerHint::Symmetric)
Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index 3c9c5fef86779..67402b4d02c66 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -48,10 +48,10 @@ using namespace llvm;
#define DEBUG_TYPE "amdgpu-pre-ra-optimizations"
-static cl::opt<bool> EnableAntiHintsForMFMARegs(
- "amdgpu-anti-hints-for-mfma", cl::Hidden,
- cl::desc("Add register allocation anti-hints for MFMA hazards."),
- cl::init(true));
+static cl::opt<bool>
+ EnableAntiHints("amdgpu-anti-hints", cl::Hidden,
+ cl::desc("Enable register allocation anti-hints."),
+ cl::init(true));
namespace {
@@ -318,7 +318,7 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
// Anti-hints only steer register allocation, so they do not count as a
// modification of the function.
- if (EnableAntiHintsForMFMARegs) {
+ if (EnableAntiHints) {
SchedModel.init(&ST);
AMDGPU::HazardContext HCtx{TII, TRI, MRI, LIS, &ST, &SchedModel};
AMDGPU::applyAntiHintRules(MF, HCtx);
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
index 148d2ced6399f..9b5f41a9fbdee 100644
--- a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-war.mir
@@ -1,10 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX90A
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX942
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX950
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX90A-NOAH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX942-NOAH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=GFX950-NOAH
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX90A
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX942
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-war %s -o - | FileCheck %s --check-prefixes=AH,AH-GFX950
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=GFX90A-NOAH
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=GFX942-NOAH
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=GFX950-NOAH
--- |
define amdgpu_kernel void @mfma_war_xdl_4pass() #0 { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
index 8404e34080096..40b98916e219b 100644
--- a/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-mfma-waw.mir
@@ -1,10 +1,10 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX90A-AH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX942-AH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=waw %s -o - | FileCheck %s --check-prefixes=AH,GFX950-AH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX90A-NOAH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX942-NOAH
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX950-NOAH
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-waw %s -o - | FileCheck %s --check-prefixes=AH,GFX90A-AH
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-waw %s -o - | FileCheck %s --check-prefixes=AH,GFX942-AH
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=mfma-waw %s -o - | FileCheck %s --check-prefixes=AH,GFX950-AH
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX90A-NOAH
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX942-NOAH
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefixes=NOAH,GFX950-NOAH
--- |
define amdgpu_kernel void @mfma_waw_nonxdl_2pass() #0 { ret void }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
index 9c948123947b5..003692e3416f6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.gfx942.mir
@@ -1,22 +1,12 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter %s -o - | FileCheck %s --check-prefix=CHECK
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter -amdgpu-anti-hints-for-mfma=false %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter -amdgpu-anti-hints=false %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
--- |
- target triple = "amdgcn-amd-amdhsa"
-
- define amdgpu_kernel void @test_anti_hints() #0 {
- bb.0:
- ret void
- }
-
- define amdgpu_kernel void @test_anti_hints_multi_mfma() #0 {
- bb.0:
- ret void
- }
+ define amdgpu_kernel void @test_anti_hints() #0 { ret void }
+ define amdgpu_kernel void @test_anti_hints_multi_mfma() #0 { ret void }
attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
-
...
---
name: test_anti_hints
>From ad83e0e7476fb0a71e52a2ce665b31f78a342481 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Thu, 24 Sep 2026 14:54:23 -0500
Subject: [PATCH 21/22] Fix leftover merge conflict.
---
llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index c27b000b6c860..67402b4d02c66 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -34,11 +34,8 @@
#include "AMDGPU.h"
#include "GCNPreRAAntiHints.h"
#include "GCNSubtarget.h"
-<<<<<<< HEAD
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
-=======
->>>>>>> users/mssefat/anti-hints-pr3-amdgpu-apply
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
>From a6fda0ada4c8b7d8a23c70d810696a3bf2655a2b Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Thu, 24 Sep 2026 17:13:28 -0500
Subject: [PATCH 22/22] [NFC] Moved getBitVecRegAntiHints from
MachineRegisterInfo to AllocationOrder
---
.../llvm/CodeGen/MachineRegisterInfo.h | 5 -----
llvm/lib/CodeGen/AllocationOrder.cpp | 20 ++++++++++++++++-
llvm/lib/CodeGen/MachineRegisterInfo.cpp | 22 -------------------
3 files changed, 19 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
index 77f75a48b5876..a5cd22f2c5d9a 100644
--- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -965,11 +965,6 @@ class MachineRegisterInfo {
return is_contained(AntiHints, AntiHintVReg);
}
- /// Get the BitVector of register units to avoid in anti-hints.
- /// VRM is the current virtual register map showing allocations made so far.
- void getBitVecRegAntiHints(Register VReg, BitVector &AntiHintedRegUnits,
- const VirtRegMap &VRM) const;
-
/// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the
/// specified register as undefined which causes the DBG_VALUE to be
/// deleted during LiveDebugVariables analysis.
diff --git a/llvm/lib/CodeGen/AllocationOrder.cpp b/llvm/lib/CodeGen/AllocationOrder.cpp
index 55be86dab2b22..6294b86b27c44 100644
--- a/llvm/lib/CodeGen/AllocationOrder.cpp
+++ b/llvm/lib/CodeGen/AllocationOrder.cpp
@@ -27,6 +27,24 @@ using namespace llvm;
#define DEBUG_TYPE "regalloc"
+static void getBitVecRegAntiHints(Register VReg, BitVector &AntiHintedRegUnits,
+ const VirtRegMap &VRM,
+ const MachineRegisterInfo &MRI,
+ const TargetRegisterInfo &TRI) {
+ assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
+ for (Register AntiHintVReg : MRI.getRegAllocationAntiHints(VReg)) {
+ // Check if the anti-hinted register has been allocated.
+ if (!VRM.hasPhys(AntiHintVReg))
+ continue;
+ // Delay until first allocated anti-hinted register so unused cases keep
+ // default empty BitVector.
+ if (AntiHintedRegUnits.empty())
+ AntiHintedRegUnits.resize(TRI.getNumRegUnits());
+ for (MCRegUnit Unit : TRI.regunits(VRM.getPhys(AntiHintVReg)))
+ AntiHintedRegUnits.set(static_cast<unsigned>(Unit));
+ }
+}
+
// Compare VirtRegMap::getRegAllocPref().
AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
const RegisterClassInfo &RegClassInfo,
@@ -57,7 +75,7 @@ AllocationOrder AllocationOrder::create(Register VirtReg, const VirtRegMap &VRM,
// Get anti-hints.
BitVector AntiHintedRegUnits;
- MRI.getBitVecRegAntiHints(VirtReg, AntiHintedRegUnits, VRM);
+ getBitVecRegAntiHints(VirtReg, AntiHintedRegUnits, VRM, MRI, *TRI);
LLVM_DEBUG({
if (AntiHintedRegUnits.any()) {
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index 61fae6d3fa281..035ccaac06a5a 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -22,7 +22,6 @@
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/DebugLoc.h"
@@ -717,24 +716,3 @@ void MachineRegisterInfo::updateDbgUsersToReg(
}
}
}
-
-void MachineRegisterInfo::getBitVecRegAntiHints(Register VReg,
- BitVector &AntiHintedRegUnits,
- const VirtRegMap &VRM) const {
- assert(VReg.isVirtual() && "Anti-hints are only for virtual registers");
- if (!AntiHintRegs.inBounds(VReg))
- return;
-
- auto *TRI = getTargetRegisterInfo();
- for (Register AntiHintVReg : AntiHintRegs[VReg]) {
- // Check if the anti-hinted register has been allocated.
- if (!VRM.hasPhys(AntiHintVReg))
- continue;
- // Delay until first allocated anti-hinted register so unused cases keep
- // default empty BitVector.
- if (AntiHintedRegUnits.empty())
- AntiHintedRegUnits.resize(TRI->getNumRegUnits());
- for (MCRegUnit Unit : TRI->regunits(VRM.getPhys(AntiHintVReg)))
- AntiHintedRegUnits.set(static_cast<unsigned>(Unit));
- }
-}
More information about the llvm-commits
mailing list