[llvm-branch-commits] [llvm] [AMDGPU] Insert wmma coexec aware anti-hints rules (PR #226397)
Syadus Sefat via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Sep 25 21:13:29 PDT 2026
https://github.com/mssefat updated https://github.com/llvm/llvm-project/pull/226397
>From bb4004f41b930037ecbdeea4621b4405cd31ed53 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Fri, 25 Sep 2026 03:39:56 -0500
Subject: [PATCH] [AMDGPU] Insert wmma coexec anti-hints rule
---
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp | 282 +-
llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h | 26 +-
.../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 60 +-
.../AMDGPU/GlobalISel/load-constant.96.ll | 20 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 24 +-
.../AMDGPU/anti-hints-addr-xcnt.gfx1250.mir | 497 ++++
.../anti-hints-multi-hazard.gfx1250.mir | 268 ++
.../AMDGPU/anti-hints-multi-rule.gfx1250.mir | 351 +++
.../anti-hints-swmmac-war-index.gfx1250.mir | 54 +
.../AMDGPU/anti-hints-trans-src0.gfx1250.mir | 166 ++
.../AMDGPU/anti-hints-va-vdst.gfx1250.mir | 137 +
.../AMDGPU/anti-hints-wmma-war.gfx1250.mir | 1610 ++++++++++++
.../AMDGPU/anti-hints-wmma-waw.gfx1250.mir | 2293 +++++++++++++++++
llvm/test/CodeGen/AMDGPU/anyext-s16-to-s64.ll | 7 +-
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 175 +-
.../AMDGPU/atomic_optimizations_buffer.ll | 24 +-
.../AMDGPU/atomic_optimizations_raw_buffer.ll | 24 +-
.../atomic_optimizations_struct_buffer.ll | 24 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 299 ++-
.../AMDGPU/branch-relaxation-gfx1250.ll | 18 +-
.../test/CodeGen/AMDGPU/carryout-selection.ll | 4 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 79 +-
llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll | 2 +-
llvm/test/CodeGen/AMDGPU/ds_write2.ll | 34 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 24 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 6 +-
.../AMDGPU/flat-load-saddr-to-vaddr.ll | 8 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 370 +--
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 60 +-
.../AMDGPU/fp-min-max-buffer-ptr-atomics.ll | 20 +-
.../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 60 +-
.../AMDGPU/fp64-min-max-buffer-ptr-atomics.ll | 10 +-
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 287 +--
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 315 ++-
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 138 +-
.../AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll | 64 +-
.../CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll | 50 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll | 4 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 68 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 246 +-
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 150 +-
.../AMDGPU/llvm.amdgcn.ptr.s.buffer.load.ll | 44 +-
.../AMDGPU/llvm.amdgcn.raw.buffer.store.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll | 36 +-
.../AMDGPU/llvm.amdgcn.struct.buffer.store.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 746 +++---
llvm/test/CodeGen/AMDGPU/load-constant-i32.ll | 171 +-
llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll | 8 +-
llvm/test/CodeGen/AMDGPU/max.ll | 24 +-
llvm/test/CodeGen/AMDGPU/min.ll | 7 +-
llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll | 26 +-
llvm/test/CodeGen/AMDGPU/sad.ll | 76 +-
llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll | 16 +-
llvm/test/CodeGen/AMDGPU/vopd3-imm-fold.ll | 16 +-
56 files changed, 7555 insertions(+), 1993 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-addr-xcnt.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-multi-hazard.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-multi-rule.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-swmmac-war-index.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-trans-src0.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-va-vdst.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-wmma-war.gfx1250.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/anti-hints-wmma-waw.gfx1250.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
index 07d565cef3fa4a..2d004692a5c2be 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.cpp
@@ -31,6 +31,12 @@ enum class AntiHintRule {
None,
MFMAWAW,
MFMAWAR,
+ WMMAWARAB,
+ SWMMACWARIndex,
+ WMMAWAW,
+ TransWAR,
+ MemAddrWAR,
+ VAVdstWAR,
All,
};
@@ -42,8 +48,45 @@ static cl::bits<AntiHintRule> AntiHintRuleSelection(
"MFMA destination write-after-write"),
clEnumValN(AntiHintRule::MFMAWAR, "mfma-war",
"XDL MFMA src2 write-after-read"),
+ clEnumValN(AntiHintRule::WMMAWARAB, "wmma-war-ab",
+ "XDL WMMA A/B source write-after-read"),
+ clEnumValN(AntiHintRule::SWMMACWARIndex, "swmmac-war-index",
+ "XDL SWMMAC sparse index write-after-read"),
+ clEnumValN(AntiHintRule::WMMAWAW, "wmma-waw",
+ "XDL WMMA destination write-after-write"),
+ clEnumValN(AntiHintRule::TransWAR, "trans-war",
+ "TRANS source write-after-read"),
+ clEnumValN(AntiHintRule::MemAddrWAR, "mem-addr-war",
+ "Memory address/data write-after-read (s_wait_xcnt)"),
+ clEnumValN(AntiHintRule::VAVdstWAR, "va-vdst-war",
+ "VALU source write-after-read by a load "
+ "(s_wait_alu va_vdst)"),
clEnumValN(AntiHintRule::All, "all",
- "Select all rules (default)")));
+ "Select all rules (default); per-rule "
+ "-amdgpu-anti-hints-for-* still apply")));
+
+static cl::opt<bool>
+ EnableAntiHintsForAddr("amdgpu-anti-hints-for-addr", cl::Hidden,
+ cl::desc("Enable Anti-Hints for memory address "
+ "operands and subsequent VGPR writes to "
+ "avoid wait xcnt."),
+ cl::init(true));
+
+static cl::opt<bool>
+ EnableAntiHintsForVAVdst("amdgpu-anti-hints-for-va-vdst", cl::Hidden,
+ cl::desc("Enable Anti-Hints for VA-VDST."),
+ cl::init(false));
+
+static cl::opt<unsigned>
+ VAVDSTLookbackWindow("amdgpu-va-vdst-lookback-window", cl::Hidden,
+ cl::desc("Lookback window for VA_VDST anti-hints."),
+ cl::init(32));
+
+static cl::opt<unsigned>
+ AddrAntiHintWindow("amdgpu-addr-anti-hint-window", cl::Hidden,
+ cl::desc("Number of later memory instructions an "
+ "address anti-hint stays open."),
+ cl::init(16));
namespace {
@@ -54,7 +97,7 @@ HazardClassMask getInstHazardClass(const MachineInstr &MI,
HazardClassMask Mask = HC::None;
if (TII.isLDSDMA(MI))
- Mask = HC::VALU | HC::VMEM | HC::DS;
+ Mask = HC::VALU | HC::VMEM | HC::DS | HC::LDSDMA;
else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
Mask = HC::WMMA;
else if (TII.isMFMA(MI))
@@ -73,6 +116,20 @@ HazardClassMask getInstHazardClass(const MachineInstr &MI,
Mask = HC::EXP;
else if (SIInstrInfo::isSALU(MI))
Mask = HC::SALU;
+ else if (MI.isCopy() && MI.getOperand(0).getReg().isVirtual() &&
+ Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MI.getOperand(0).getReg())))
+ Mask = HC::VALU;
+ if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
+ Mask |= HC::RawVALU;
+ if (MI.mayLoad())
+ Mask |= HC::Load;
+ for (const MachineOperand &MO : MI.defs()) {
+ if (MO.isReg() && MO.getReg().isVirtual() &&
+ Ctx.TRI->hasVGPRs(Ctx.MRI->getRegClass(MO.getReg()))) {
+ Mask |= HC::WritesVGPR;
+ break;
+ }
+ }
return Mask;
}
@@ -104,6 +161,10 @@ void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
case HazardOperand::Src2:
Named(AMDGPU::OpName::src2);
break;
+ case HazardOperand::Src0Src1:
+ Named(AMDGPU::OpName::src0);
+ Named(AMDGPU::OpName::src1);
+ break;
case HazardOperand::Idx:
Named(AMDGPU::OpName::idx);
break;
@@ -199,13 +260,48 @@ unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
return 0;
}
-unsigned mfmaWawWindow(const MachineInstr &P, const HazardContext &Ctx) {
+unsigned mfmaWawWindow(const MachineInstr &P, HazardClassMask,
+ const HazardContext &Ctx) {
return mfmaWaitStates(P, MFMAHazardKind::WAW, HC::None, Ctx);
}
-unsigned mfmaWarWindow(const MachineInstr &P, const HazardContext &Ctx) {
+unsigned mfmaWarWindow(const MachineInstr &P, HazardClassMask,
+ const HazardContext &Ctx) {
return mfmaWaitStates(P, MFMAHazardKind::WAR, HC::None, Ctx);
}
+constexpr unsigned NumWMMAHazardCategories = 7;
+unsigned wmmaHazardCategory(const MachineInstr &Producer,
+ const HazardContext &Ctx) {
+ const bool IsSWMMAC = SIInstrInfo::isSWMMAC(Producer);
+ const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA();
+ switch (Ctx.SchedModel->computeInstrLatency(&Producer)) {
+ case 4:
+ return 6;
+ case 8:
+ return IsSWMMAC ? 2 : 0;
+ case 16:
+ return LowestRate ? 4 : (IsSWMMAC ? 3 : 1);
+ case 32:
+ return 5;
+ default:
+ return NumWMMAHazardCategories;
+ }
+}
+
+// WMMA/SWMMAC co-exec window.
+unsigned wmmaCoexecWindow(const MachineInstr &Producer,
+ HazardClassMask ConsumerClass,
+ const HazardContext &Ctx) {
+ const unsigned Category = wmmaHazardCategory(Producer, Ctx);
+ if (Category >= NumWMMAHazardCategories)
+ return 0;
+
+ static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
+ static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
+ return (ConsumerClass & HC::WMMA) ? WMMAWaitStates[Category]
+ : VALUWaitStates[Category];
+}
+
unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
HazardClassMask ReaderClass,
const HazardContext &Ctx) {
@@ -216,7 +312,20 @@ bool hasMFMAHazard(const HazardContext &Ctx) {
return Ctx.ST->hasGFX90AInsts();
}
+bool hasWMMACoexecHazard(const HazardContext &Ctx) {
+ return Ctx.ST->hasWMMACoexecutionHazards();
+}
+
+bool hasTransCoexecHazard(const HazardContext &Ctx) {
+ return Ctx.ST->hasTransCoexecutionHazard();
+}
+
+bool hasGFX1250Insts(const HazardContext &Ctx) {
+ return Ctx.ST->hasGFX1250Insts();
+}
+
bool ruleSelected(AntiHintRule Rule) {
+ // No -amdgpu-anti-hints-rules on the command line selects every rule.
if (!AntiHintRuleSelection.getNumOccurrences() ||
AntiHintRuleSelection.isSet(AntiHintRule::All))
return true;
@@ -231,18 +340,55 @@ bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
return hasMFMAHazard(Ctx) && ruleSelected(AntiHintRule::MFMAWAR);
}
+bool isWMMAWARABRuleEnabled(const HazardContext &Ctx) {
+ return hasWMMACoexecHazard(Ctx) && ruleSelected(AntiHintRule::WMMAWARAB);
+}
+
+bool isSWMMACWARIndexRuleEnabled(const HazardContext &Ctx) {
+ return hasWMMACoexecHazard(Ctx) && ruleSelected(AntiHintRule::SWMMACWARIndex);
+}
+
+bool isWMMAWAWRuleEnabled(const HazardContext &Ctx) {
+ return hasWMMACoexecHazard(Ctx) && ruleSelected(AntiHintRule::WMMAWAW);
+}
+
+bool isTransWARRuleEnabled(const HazardContext &Ctx) {
+ return hasTransCoexecHazard(Ctx) && ruleSelected(AntiHintRule::TransWAR);
+}
+
+bool isAddrWARRuleEnabled(const HazardContext &Ctx) {
+ return EnableAntiHintsForAddr && hasGFX1250Insts(Ctx) &&
+ ruleSelected(AntiHintRule::MemAddrWAR);
+}
+
+bool isVAVdstWARRuleEnabled(const HazardContext &Ctx) {
+ return EnableAntiHintsForVAVdst && hasGFX1250Insts(Ctx) &&
+ ruleSelected(AntiHintRule::VAVdstWAR);
+}
+
bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
return Ctx.TII->isXDL(MI);
}
+bool isXDLWMMA(const MachineInstr &MI, const HazardContext &Ctx) {
+ return Ctx.TII->isXDLWMMA(MI);
+}
+
+bool isXDLSWMMAC(const MachineInstr &MI, const HazardContext &Ctx) {
+ return Ctx.TII->isXDLWMMA(MI) && SIInstrInfo::isSWMMAC(MI);
+}
+
+bool isDSorVMEMLoad(const MachineInstr &MI, const HazardContext &Ctx) {
+ return MI.mayLoad() && (Ctx.TII->isDS(MI) || Ctx.TII->isVMEM(MI));
+}
+
unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
const HazardContext &Ctx) {
// Explicity given window length overrides the computed one.
- if (CT.Window.OptWindowLength &&
- CT.Window.OptWindowLength->getNumOccurrences())
+ if (CT.Window.OptWindowLength)
return *CT.Window.OptWindowLength;
if (CT.Window.Fn)
- return CT.Window.Fn(MI, Ctx);
+ return CT.Window.Fn(MI, CT.Side.Match.AnyOf, Ctx);
return CT.Window.WindowLength;
}
@@ -271,11 +417,13 @@ class HazardRuleSet {
return *this;
}
- RuleBuilder &consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
- HazardClassMask CountMask = 0,
- ConsumerHint Hint = ConsumerHint::OneDirectional,
- InstPredicate Predicate = nullptr) {
- rule().Consumers.push_back({{M, Op, Predicate}, Window, CountMask, Hint});
+ RuleBuilder &
+ consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
+ HazardClassMask CountMask = 0,
+ AntiHintDirection Direction = AntiHintDirection::OneDirectional,
+ InstPredicate Predicate = nullptr) {
+ rule().Consumers.push_back(
+ {{M, Op, Predicate}, Window, CountMask, Direction});
return *this;
}
@@ -283,6 +431,11 @@ class HazardRuleSet {
rule().Predicate = Predicate;
return *this;
}
+
+ RuleBuilder &lifetime(Lifetime Life) {
+ rule().Life = Life;
+ return *this;
+ }
};
RuleBuilder addRule() {
@@ -298,18 +451,26 @@ SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
using HO = HazardOperand;
HazardRuleSet S;
+ // MFMA relevent windows and consumers
const WindowSpec MfmaWawWindow{0, nullptr, mfmaWawWindow};
const WindowSpec MfmaWarWindow{0, nullptr, mfmaWarWindow};
const ClassMatch MfmaConsumers = {HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
HC::EXP};
+ // WMMA relevent windows and consumers
+ const WindowSpec WmmaCoexecWindow{0, nullptr, wmmaCoexecWindow};
+ const ClassMatch WmmaCoexecConsumers = {/*AnyOf=*/HC::VALU | HC::TRANS,
+ /*AllOf=*/HC::None,
+ /*NoneOf=*/HC::LDSDMA};
+ const HazardClassMask CoexecCounters = HC::VALU | HC::TRANS | HC::WMMA;
+
// MFMA WAW rules
S.addRule()
.enabledIf(isMFMAWAWRuleEnabled)
.producer({HC::MFMA}, HO::Def)
.rawCredit(mfmaReaderRawWindow)
.consumer(MfmaConsumers, HO::Def, MfmaWawWindow, HC::None,
- ConsumerHint::OneDirectional);
+ AntiHintDirection::OneDirectional);
// MFMA WAR rules
S.addRule()
@@ -317,7 +478,52 @@ SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
.producer({HC::MFMA}, HO::Src2, isXDLMFMA)
.rawCredit(mfmaReaderRawWindow)
.consumer(MfmaConsumers, HO::Def, MfmaWarWindow, HC::None,
- ConsumerHint::OneDirectional);
+ AntiHintDirection::OneDirectional);
+
+ // WMMA WAR.
+ S.addRule()
+ .enabledIf(isWMMAWARABRuleEnabled)
+ .producer({HC::WMMA}, HO::Src0Src1, isXDLWMMA)
+ .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
+ AntiHintDirection::OneDirectional)
+ .consumer({HC::VMEM | HC::DS}, HO::Def, WmmaCoexecWindow, CoexecCounters,
+ AntiHintDirection::OneDirectional);
+
+ // SWMMAC WAR.
+ S.addRule()
+ .enabledIf(isSWMMACWARIndexRuleEnabled)
+ .producer({HC::WMMA}, HO::Src2, isXDLSWMMAC)
+ .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
+ AntiHintDirection::OneDirectional);
+
+ // WMMA WAW.
+ S.addRule()
+ .enabledIf(isWMMAWAWRuleEnabled)
+ .producer({HC::WMMA}, HO::Def, isXDLWMMA)
+ .consumer(WmmaCoexecConsumers, HO::Def, WmmaCoexecWindow, CoexecCounters,
+ AntiHintDirection::OneDirectional);
+
+ // TRANS WAR.
+ S.addRule()
+ .enabledIf(isTransWARRuleEnabled)
+ .producer({HC::TRANS}, HO::AnyUse)
+ .consumer({HC::VALU | HC::WMMA}, HO::Def, {/*WindowLength=*/1},
+ CoexecCounters, AntiHintDirection::OneDirectional);
+
+ // Address WAR.
+ S.addRule()
+ .enabledIf(isAddrWARRuleEnabled)
+ .producer({HC::VMEM}, HO::AnyUse)
+ .consumer({HC::WritesVGPR}, HO::Def, {0, &AddrAntiHintWindow}, HC::VMEM,
+ AntiHintDirection::Symmetric);
+
+ // VA_VDST WAR.
+ S.addRule()
+ .enabledIf(isVAVdstWARRuleEnabled)
+ .lifetime(Lifetime::RegisterCount)
+ .producer({HC::RawVALU}, HO::AnyUse)
+ .consumer({HC::Load}, HO::Def, {0, &VAVDSTLookbackWindow}, HC::None,
+ AntiHintDirection::Symmetric, isDSorVMEMLoad);
return S.buildRules();
}
@@ -401,13 +607,35 @@ class AntiHintEngine {
if (Regs.empty())
continue;
for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
- ConsumerIdx != E; ++ConsumerIdx) {
- unsigned Window = resolveWindow(Rule.Consumers[ConsumerIdx], MI, Ctx);
- if (!Window)
+ ConsumerIdx != E; ++ConsumerIdx)
+ seedWindow(Rule, Rule.Consumers[ConsumerIdx], MI, Regs,
+ Tracking[R][ConsumerIdx]);
+ }
+ }
+
+ void seedWindow(const HazardAntiHintRule &Rule, const ConsumerTarget &CT,
+ const MachineInstr &MI, ArrayRef<Register> Regs,
+ ConsumerTracking &Track) {
+ unsigned Window = resolveWindow(CT, MI, Ctx);
+ if (!Window)
+ return;
+
+ if (Rule.Life == Lifetime::RegisterCount) {
+ // Window length caps the register count.
+ if (Track.empty())
+ Track.push_back({{}, &MI, Window, 0});
+ SmallVectorImpl<Register> &Recent = Track.front().Regs;
+ for (Register Reg : Regs) {
+ if (llvm::is_contained(Recent, Reg))
continue;
- Tracking[R][ConsumerIdx].push_back({Regs, &MI, Window, 0});
+ Recent.push_back(Reg);
+ if (Recent.size() > Window)
+ Recent.erase(Recent.begin());
}
+ return;
}
+
+ Track.push_back({SmallVector<Register, 4>(Regs), &MI, Window, 0});
}
void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
@@ -448,17 +676,20 @@ class AntiHintEngine {
// Before adding the anti-hints, see if advancing by RAW window will
// help remove the window.
- advanceByRawWindow(MI, C, Rule, CT, Track);
- llvm::erase_if(Track, [](const AntiHintWindow &Window) {
- return Window.Elapsed >= Window.Len;
- });
+ const bool Counts = Rule.Life == Lifetime::WindowBudget;
+ if (Counts) {
+ advanceByRawWindow(MI, C, Rule, CT, Track);
+ llvm::erase_if(Track, [](const AntiHintWindow &Window) {
+ return Window.Elapsed >= Window.Len;
+ });
+ }
// Add anti-hints if the consumer matches the instruction.
if (sideMatches(CT.Side, MI, C))
addAntiHints(CT, MI, Track);
// This instruction's own wait states count toward the next one.
- if (!CT.CounterMask || (C & CT.CounterMask))
+ if (Counts && (!CT.CounterMask || (C & CT.CounterMask)))
for (AntiHintWindow &Window : Track)
Window.Elapsed += WaitStates;
}
@@ -492,11 +723,12 @@ class AntiHintEngine {
continue;
Ctx.MRI->addRegAllocationAntiHints(ConsumerReg, ProducerReg);
- if (CT.Hint == ConsumerHint::Symmetric)
+ if (CT.Direction == AntiHintDirection::Symmetric)
Ctx.MRI->addRegAllocationAntiHints(ProducerReg, ConsumerReg);
LLVM_DEBUG(
dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
- << (CT.Hint == ConsumerHint::Symmetric ? " <-> " : " <- ")
+ << (CT.Direction == AntiHintDirection::Symmetric ? " <-> "
+ : " <- ")
<< printReg(ConsumerReg, Ctx.TRI) << " (consumer "
<< Ctx.TII->getName(MI.getOpcode()) << ")\n");
}
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
index e9b0e884d549b4..1aa8f32cf8379f 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAAntiHints.h
@@ -41,6 +41,11 @@ enum : HazardClassMask {
WMMA = 1u << 7,
MFMA = 1u << 8,
EXP = 1u << 9,
+
+ Load = 1u << 10,
+ WritesVGPR = 1u << 11,
+ RawVALU = 1u << 12,
+ LDSDMA = 1u << 13,
};
} // namespace HazardClass
@@ -50,17 +55,24 @@ enum class HazardOperand : uint8_t {
Src0,
Src1,
Src2,
+ Src0Src1,
Idx,
Vaddr,
AnySrc,
AnyUse,
};
-enum class ConsumerHint : uint8_t {
+enum class AntiHintDirection : uint8_t {
OneDirectional,
Symmetric,
};
+enum class Lifetime : uint8_t {
+ // Wait-state budget.
+ WindowBudget,
+ // Most recent producer regs count cap.
+ RegisterCount,
+};
struct HazardContext {
const SIInstrInfo *TII;
const SIRegisterInfo *TRI;
@@ -72,7 +84,12 @@ struct HazardContext {
struct ClassMatch {
HazardClassMask AnyOf = 0;
- bool matches(HazardClassMask Mask) const { return !AnyOf || (Mask & AnyOf); }
+ HazardClassMask AllOf = 0;
+ HazardClassMask NoneOf = 0;
+ bool matches(HazardClassMask Mask) const {
+ return (!AnyOf || (Mask & AnyOf)) && ((Mask & AllOf) == AllOf) &&
+ !(Mask & NoneOf);
+ }
};
// To enable or disable the whole rule.
@@ -98,7 +115,7 @@ struct HazardSide {
struct WindowSpec {
unsigned WindowLength = 0;
const cl::opt<unsigned> *OptWindowLength = nullptr;
- unsigned (*Fn)(const MachineInstr &Producer,
+ unsigned (*Fn)(const MachineInstr &Producer, HazardClassMask ConsumerClass,
const HazardContext &Ctx) = nullptr;
};
@@ -106,11 +123,12 @@ struct ConsumerTarget {
HazardSide Side;
WindowSpec Window;
HazardClassMask CounterMask = HazardClass::None;
- ConsumerHint Hint = ConsumerHint::OneDirectional;
+ AntiHintDirection Direction = AntiHintDirection::OneDirectional;
};
struct HazardAntiHintRule {
RulePredicate Predicate = nullptr;
+ Lifetime Life = Lifetime::WindowBudget;
HazardSide Producer;
SmallVector<ConsumerTarget, 3> Consumers;
AdvanceForRawWindowFn AdvanceForRawWindow = nullptr;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index a8942ab52166c9..b43b941190b9b9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -131,15 +131,14 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -262,15 +261,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -392,15 +390,14 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -522,15 +519,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -653,15 +649,14 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -784,15 +779,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -914,15 +908,14 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1044,15 +1037,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1175,15 +1167,14 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -1306,15 +1297,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -1436,15 +1426,14 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1566,15 +1555,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 540be44df1bfd9..2591d2afbb7d43 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -92,22 +92,22 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v12, v[0:1], off offset:11 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v13, v[0:1], off offset:10 nv
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0xa
-; GFX1250-NOUNALIGNED-NEXT: s_wait_xcnt 0x0
-; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v0, v3, 8, v2
+; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v2, v3, 8, v2
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x8
-; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v2, 16, v4 :: v_dual_lshlrev_b32 v1, 24, v5
+; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v4, 16, v4 :: v_dual_lshlrev_b32 v3, 24, v5
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x6
-; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v5, v7, 8, v6
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x4
-; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v5, 16, v8 :: v_dual_lshlrev_b32 v4, 24, v9
-; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v7, 16, v8 :: v_dual_lshlrev_b32 v6, 24, v9
+; GFX1250-NOUNALIGNED-NEXT: s_wait_xcnt 0x0
+; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v0, v3, v4, v2
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x2
-; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
+; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v8, v11, 8, v10
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v7, 24, v12 :: v_dual_lshlrev_b32 v8, 16, v13
-; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v1, v4, v5, v3
+; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v9, 24, v12 :: v_dual_lshlrev_b32 v10, 16, v13
+; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v1, v6, v7, v5
; GFX1250-NOUNALIGNED-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v2, v7, v8, v6
+; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v2, v9, v10, v8
; GFX1250-NOUNALIGNED-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9-UNALIGNED-LABEL: v_load_constant_v3i32_align1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 8b5e9d45a08bcb..1e3130e9a240da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -3842,18 +3842,18 @@ define amdgpu_ps void @s_mul_u64_zext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[2:3], 0x50, v4, 0
-; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[4:5], 0x50, v2, 0
+; GFX1250-NEXT: global_store_b64 v[0:1], v[4:5], off
; GFX1250-NEXT: s_endpgm
;
; GFX13-LABEL: s_mul_u64_zext_with_vregs:
; GFX13: ; %bb.0:
-; GFX13-NEXT: global_load_b32 v4, v[2:3], off
+; GFX13-NEXT: global_load_b32 v2, v[2:3], off
; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: v_mad_co_u64_u32 v[2:3], null, 0x50, v4, 0
-; GFX13-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX13-NEXT: v_mad_co_u64_u32 v[4:5], null, 0x50, v2, 0
+; GFX13-NEXT: global_store_b64 v[0:1], v[4:5], off
; GFX13-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
%ext = zext i32 %val to i64
@@ -4066,18 +4066,18 @@ define amdgpu_ps void @s_mul_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mad_nc_i64_i32 v[2:3], 0x50, v4, 0
-; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[4:5], 0x50, v2, 0
+; GFX1250-NEXT: global_store_b64 v[0:1], v[4:5], off
; GFX1250-NEXT: s_endpgm
;
; GFX13-LABEL: s_mul_u64_sext_with_vregs:
; GFX13: ; %bb.0:
-; GFX13-NEXT: global_load_b32 v4, v[2:3], off
+; GFX13-NEXT: global_load_b32 v2, v[2:3], off
; GFX13-NEXT: s_wait_loadcnt 0x0
-; GFX13-NEXT: v_mad_co_i64_i32 v[2:3], null, 0x50, v4, 0
-; GFX13-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX13-NEXT: v_mad_co_i64_i32 v[4:5], null, 0x50, v2, 0
+; GFX13-NEXT: global_store_b64 v[0:1], v[4:5], off
; GFX13-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %in, align 4
%ext = sext i32 %val to i64
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-addr-xcnt.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-addr-xcnt.gfx1250.mir
new file mode 100644
index 00000000000000..e1134b4781f9c3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-addr-xcnt.gfx1250.mir
@@ -0,0 +1,497 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-for-addr=true -amdgpu-anti-hints-rules=mem-addr-war %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=NOAH
+
+--- |
+ define amdgpu_kernel void @addr_vaddr_valu_def() #0 { ret void }
+ define amdgpu_kernel void @addr_store_vdata_not_vaddr() #0 { ret void }
+ define amdgpu_kernel void @addr_xcnt() #0 { ret void }
+ define amdgpu_kernel void @addr_window_overflow() #0 { ret void }
+ define amdgpu_kernel void @addr_reread_no_refresh() #0 { ret void }
+ define amdgpu_kernel void @addr_fallthrough_block() #0 { ret void }
+ define amdgpu_kernel void @addr_vmem_consumer_vmvsrc() #0 { ret void }
+ define amdgpu_kernel void @addr_ds_producer_vmvsrc() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+---
+name: addr_vaddr_valu_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10
+
+ ; CHECK-LABEL: name: addr_vaddr_valu_def
+ ; CHECK: liveins: $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 2, killed $vgpr10, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: addr_vaddr_valu_def
+ ; NOAH: liveins: $vgpr10
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAIT_XCNT 0
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 2, killed $vgpr10, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = COPY $vgpr10
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD %0, 0, 0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2, implicit %3
+...
+
+---
+name: addr_store_vdata_not_vaddr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: addr_store_vdata_not_vaddr
+ ; CHECK: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 1, $vgpr3, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4
+ ;
+ ; NOAH-LABEL: name: addr_store_vdata_not_vaddr
+ ; NOAH: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAIT_XCNT 0
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr3, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr3, implicit killed renamable $vgpr2
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 1, %2, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %2, implicit %3
+...
+
+---
+name: addr_xcnt
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' }
+ - { reg: '$vgpr4_vgpr5_vgpr6_vgpr7' }
+ - { reg: '$vgpr8_vgpr9_vgpr10_vgpr11' }
+ - { reg: '$vgpr12_vgpr13_vgpr14_vgpr15' }
+ - { reg: '$vgpr16' }
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+ ; CHECK-LABEL: name: addr_xcnt
+ ; CHECK: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFEN renamable $vgpr4_vgpr5_vgpr6_vgpr7, killed renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 2, $vgpr1, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 3, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: addr_xcnt
+ ; NOAH: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: BUFFER_STORE_DWORDX4_OFFEN renamable $vgpr4_vgpr5_vgpr6_vgpr7, killed renamable $vgpr16, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; NOAH-NEXT: S_WAIT_XCNT 0
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr1, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr2, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vreg_128_align2 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+ %2:vreg_128_align2 = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+ %3:vreg_128_align2 = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+ %4:vgpr_32 = COPY $vgpr16
+ %6:sgpr_128 = IMPLICIT_DEF
+ BUFFER_STORE_DWORDX4_OFFEN %1, %4, %6, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ %8:vgpr_32 = V_ADD_U32_e32 1, %0.sub0, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 2, %0.sub1, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 3, %0.sub2, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: addr_window_overflow
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10
+ ; CHECK-LABEL: name: addr_window_overflow
+ ; CHECK: liveins: $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr11 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr21 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr25 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr26 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr18_vgpr19 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr11, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr2_vgpr3, killed renamable $vgpr20, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr4_vgpr5, killed renamable $vgpr21, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr6_vgpr7, killed renamable $vgpr22, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr8_vgpr9, killed renamable $vgpr23, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr12_vgpr13, killed renamable $vgpr24, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr14_vgpr15, killed renamable $vgpr25, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr16_vgpr17, killed renamable $vgpr26, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr18_vgpr19, killed renamable $vgpr27, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13, implicit killed renamable $vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17, implicit killed renamable $vgpr18_vgpr19
+ ;
+ ; NOAH-LABEL: name: addr_window_overflow
+ ; NOAH: liveins: $vgpr10
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr11 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr21 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr25 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr26 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr27 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr18_vgpr19 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr11, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr2_vgpr3, killed renamable $vgpr20, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr4_vgpr5, killed renamable $vgpr21, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr6_vgpr7, killed renamable $vgpr22, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr8_vgpr9, killed renamable $vgpr23, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr12_vgpr13, killed renamable $vgpr24, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr14_vgpr15, killed renamable $vgpr25, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr16_vgpr17, killed renamable $vgpr26, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr18_vgpr19, killed renamable $vgpr27, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAIT_XCNT 8
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_6
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13, implicit killed renamable $vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17, implicit killed renamable $vgpr18_vgpr19
+ %live:vgpr_32 = COPY $vgpr10
+ %d1:vgpr_32 = IMPLICIT_DEF
+ %d2:vgpr_32 = IMPLICIT_DEF
+ %d3:vgpr_32 = IMPLICIT_DEF
+ %d4:vgpr_32 = IMPLICIT_DEF
+ %d5:vgpr_32 = IMPLICIT_DEF
+ %d6:vgpr_32 = IMPLICIT_DEF
+ %d7:vgpr_32 = IMPLICIT_DEF
+ %d8:vgpr_32 = IMPLICIT_DEF
+ %d9:vgpr_32 = IMPLICIT_DEF
+ %a1:vreg_64_align2 = IMPLICIT_DEF
+ %a2:vreg_64_align2 = IMPLICIT_DEF
+ %a3:vreg_64_align2 = IMPLICIT_DEF
+ %a4:vreg_64_align2 = IMPLICIT_DEF
+ %a5:vreg_64_align2 = IMPLICIT_DEF
+ %a6:vreg_64_align2 = IMPLICIT_DEF
+ %a7:vreg_64_align2 = IMPLICIT_DEF
+ %a8:vreg_64_align2 = IMPLICIT_DEF
+ %a9:vreg_64_align2 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %a1, %d1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a2, %d2, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a3, %d3, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a4, %d4, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a5, %d5, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a6, %d6, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a7, %d7, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a8, %d8, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a9, %d9, 0, 0, implicit $exec
+ %r:vgpr_32 = V_ADD_U32_e32 1, %live, implicit $exec
+ S_ENDPGM 0, implicit %live, implicit %r, implicit %a1, implicit %a2, implicit %a3, implicit %a4, implicit %a5, implicit %a6, implicit %a7, implicit %a8, implicit %a9
+...
+
+---
+name: addr_reread_no_refresh
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10
+ ; CHECK-LABEL: name: addr_reread_no_refresh
+ ; CHECK: liveins: $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr11 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr25 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr26 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr18_vgpr19 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr20_vgpr21 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, renamable $vgpr11, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr2_vgpr3, killed renamable $vgpr22, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr4_vgpr5, killed renamable $vgpr23, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr6_vgpr7, killed renamable $vgpr24, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr8_vgpr9, killed renamable $vgpr25, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr12_vgpr13, killed renamable $vgpr26, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr14_vgpr15, killed renamable $vgpr27, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr16_vgpr17, killed renamable $vgpr11, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr18_vgpr19, killed renamable $vgpr28, 0, 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr20_vgpr21, killed renamable $vgpr29, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10, implicit killed renamable $vgpr30, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13, implicit killed renamable $vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17, implicit killed renamable $vgpr18_vgpr19, implicit killed renamable $vgpr20_vgpr21
+ ;
+ ; NOAH-LABEL: name: addr_reread_no_refresh
+ ; NOAH: liveins: $vgpr10
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr11 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr25 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr26 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr27 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr18_vgpr19 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr20_vgpr21 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, renamable $vgpr11, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr2_vgpr3, killed renamable $vgpr22, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr4_vgpr5, killed renamable $vgpr23, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr6_vgpr7, killed renamable $vgpr24, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr8_vgpr9, killed renamable $vgpr25, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr12_vgpr13, killed renamable $vgpr26, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr14_vgpr15, killed renamable $vgpr27, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr16_vgpr17, killed renamable $vgpr11, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr18_vgpr19, killed renamable $vgpr28, 0, 0, implicit $exec
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr20_vgpr21, killed renamable $vgpr29, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAIT_XCNT 2
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_2
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr12_vgpr13, implicit killed renamable $vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17, implicit killed renamable $vgpr18_vgpr19, implicit killed renamable $vgpr20_vgpr21
+ %live:vgpr_32 = COPY $vgpr10
+ %d1:vgpr_32 = IMPLICIT_DEF
+ %d2:vgpr_32 = IMPLICIT_DEF
+ %d3:vgpr_32 = IMPLICIT_DEF
+ %d4:vgpr_32 = IMPLICIT_DEF
+ %d5:vgpr_32 = IMPLICIT_DEF
+ %d6:vgpr_32 = IMPLICIT_DEF
+ %d7:vgpr_32 = IMPLICIT_DEF
+ %d9:vgpr_32 = IMPLICIT_DEF
+ %d10:vgpr_32 = IMPLICIT_DEF
+ %a1:vreg_64_align2 = IMPLICIT_DEF
+ %a2:vreg_64_align2 = IMPLICIT_DEF
+ %a3:vreg_64_align2 = IMPLICIT_DEF
+ %a4:vreg_64_align2 = IMPLICIT_DEF
+ %a5:vreg_64_align2 = IMPLICIT_DEF
+ %a6:vreg_64_align2 = IMPLICIT_DEF
+ %a7:vreg_64_align2 = IMPLICIT_DEF
+ %a8:vreg_64_align2 = IMPLICIT_DEF
+ %a9:vreg_64_align2 = IMPLICIT_DEF
+ %a10:vreg_64_align2 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %a1, %d1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a2, %d2, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a3, %d3, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a4, %d4, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a5, %d5, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a6, %d6, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a7, %d7, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a8, %d1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a9, %d9, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORD %a10, %d10, 0, 0, implicit $exec
+ %r:vgpr_32 = V_ADD_U32_e32 1, %live, implicit $exec
+ S_ENDPGM 0, implicit %live, implicit %r, implicit %a1, implicit %a2, implicit %a3, implicit %a4, implicit %a5, implicit %a6, implicit %a7, implicit %a8, implicit %a9, implicit %a10
+...
+
+---
+name: addr_fallthrough_block
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: addr_fallthrough_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr3, 0, 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: liveins: $vgpr2, $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_WAIT_XCNT 0
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: addr_fallthrough_block
+ ; NOAH: bb.0:
+ ; NOAH-NEXT: successors: %bb.1(0x80000000)
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr3, 0, 0, implicit $exec
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: bb.1:
+ ; NOAH-NEXT: liveins: $vgpr2, $vgpr0_vgpr1
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: S_WAIT_XCNT 0
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ bb.0:
+ successors: %bb.1
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+
+ bb.1:
+ %3:vgpr_32 = V_ADD_U32_e32 1, %2, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %2, implicit %3
+...
+
+---
+name: addr_vmem_consumer_vmvsrc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: addr_vmem_consumer_vmvsrc
+ ; CHECK: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr4, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = GLOBAL_LOAD_DWORD renamable $vgpr2_vgpr3, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr5
+ ;
+ ; NOAH-LABEL: name: addr_vmem_consumer_vmvsrc
+ ; NOAH: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: GLOBAL_STORE_DWORD renamable $vgpr0_vgpr1, killed renamable $vgpr4, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr4 = GLOBAL_LOAD_DWORD renamable $vgpr2_vgpr3, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr4
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+ %3:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %2, implicit %3
+...
+
+---
+name: addr_ds_producer_vmvsrc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: addr_ds_producer_vmvsrc
+ ; CHECK: renamable $vgpr0 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr1
+ ;
+ ; NOAH-LABEL: name: addr_ds_producer_vmvsrc
+ ; NOAH: renamable $vgpr0 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: DS_WRITE_B32_gfx9 renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr1
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ DS_WRITE_B32_gfx9 %0, %1, 0, 0, implicit $exec
+ %3:vgpr_32 = V_ADD_U32_e32 1, %2, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %2, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-multi-hazard.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-multi-hazard.gfx1250.mir
new file mode 100644
index 00000000000000..f04f917e2aec8e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-multi-hazard.gfx1250.mir
@@ -0,0 +1,268 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-for-va-vdst=true -amdgpu-anti-hints-for-addr=true %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
+
+--- |
+ define amdgpu_kernel void @two_wmma_overlapping() #0 { ret void }
+ define amdgpu_kernel void @wmma_multi_consumer_slots() #0 { ret void }
+ define amdgpu_kernel void @trans_plus_wmma_interleave() #0 { ret void }
+ define amdgpu_kernel void @vavdst_multi_producer() #0 { ret void }
+ define amdgpu_kernel void @addr_multi_producer() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+---
+name: two_wmma_overlapping
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr40
+
+ ; CHECK-LABEL: name: two_wmma_overlapping
+ ; CHECK: liveins: $vgpr40
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 4, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: two_wmma_overlapping
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr40
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 4, killed $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ %s:vgpr_32 = COPY $vgpr40
+ %a0:vreg_256_align2 = IMPLICIT_DEF
+ %a1:vreg_256_align2 = IMPLICIT_DEF
+ %a2:vreg_256_align2 = IMPLICIT_DEF
+ %ad:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %a0, %a1, 8, %a2, 0, 0, 0, 0, implicit $exec
+ %c1:vgpr_32 = V_ADD_U32_e32 1, %s, implicit $exec
+ %c2:vgpr_32 = V_ADD_U32_e32 2, %s, implicit $exec
+ %b0:vreg_256_align2 = IMPLICIT_DEF
+ %b1:vreg_256_align2 = IMPLICIT_DEF
+ %b2:vreg_256_align2 = IMPLICIT_DEF
+ %bd:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %b0, %b1, 8, %b2, 0, 0, 0, 0, implicit $exec
+ %c3:vgpr_32 = V_ADD_U32_e32 3, %s, implicit $exec
+ %c4:vgpr_32 = V_ADD_U32_e32 4, %s, implicit $exec
+ S_ENDPGM 0, implicit %ad, implicit %bd, implicit %c1, implicit %c2, implicit %c3, implicit %c4
+...
+
+---
+name: wmma_multi_consumer_slots
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr40
+
+ ; CHECK-LABEL: name: wmma_multi_consumer_slots
+ ; CHECK: liveins: $vgpr40
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: wmma_multi_consumer_slots
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr40
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, killed $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2
+ %s:vgpr_32 = COPY $vgpr40
+ %a0:vreg_256_align2 = IMPLICIT_DEF
+ %a1:vreg_256_align2 = IMPLICIT_DEF
+ %a2:vreg_256_align2 = IMPLICIT_DEF
+ %ad:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %a0, %a1, 8, %a2, 0, 0, 0, 0, implicit $exec
+ %c1:vgpr_32 = V_ADD_U32_e32 1, %s, implicit $exec
+ %c2:vgpr_32 = V_ADD_U32_e32 2, %s, implicit $exec
+ %c3:vgpr_32 = V_ADD_U32_e32 3, %s, implicit $exec
+ S_ENDPGM 0, implicit %ad, implicit %c1, implicit %c2, implicit %c3
+...
+
+---
+name: trans_plus_wmma_interleave
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr40, $vgpr41
+
+ ; CHECK-LABEL: name: trans_plus_wmma_interleave
+ ; CHECK: liveins: $vgpr40, $vgpr41
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 killed $vgpr41, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: trans_plus_wmma_interleave
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr40, $vgpr41
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 killed $vgpr41, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, killed $vgpr40, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ %s:vgpr_32 = COPY $vgpr40
+ %t0:vgpr_32 = COPY $vgpr41
+ %tr:vgpr_32 = nofpexcept V_EXP_F32_e32 %t0, implicit $mode, implicit $exec
+ %c1:vgpr_32 = V_ADD_U32_e32 1, %s, implicit $exec
+ %a0:vreg_256_align2 = IMPLICIT_DEF
+ %a1:vreg_256_align2 = IMPLICIT_DEF
+ %a2:vreg_256_align2 = IMPLICIT_DEF
+ %ad:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %a0, %a1, 8, %a2, 0, 0, 0, 0, implicit $exec
+ %c2:vgpr_32 = V_ADD_U32_e32 2, %s, implicit $exec
+ %c3:vgpr_32 = V_ADD_U32_e32 3, %s, implicit $exec
+ S_ENDPGM 0, implicit %tr, implicit %ad, implicit %c1, implicit %c2, implicit %c3
+...
+
+---
+name: vavdst_multi_producer
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10, $vgpr11, $vgpr12
+
+ ; CHECK-LABEL: name: vavdst_multi_producer
+ ; CHECK: liveins: $vgpr10, $vgpr11, $vgpr12
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, killed $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, killed $vgpr11, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr12, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = DS_READ_B128_gfx9 killed renamable $vgpr12, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: vavdst_multi_producer
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr10, $vgpr11, $vgpr12
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, killed $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, killed $vgpr11, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr12, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = DS_READ_B128_gfx9 killed renamable $vgpr12, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7
+ %p0:vgpr_32 = COPY $vgpr10
+ %p1:vgpr_32 = COPY $vgpr11
+ %addr:vgpr_32 = COPY $vgpr12
+ %v1:vgpr_32 = V_ADD_U32_e32 1, %p0, implicit $exec
+ %v2:vgpr_32 = V_ADD_U32_e32 2, %p1, implicit $exec
+ %l1:vreg_128_align2 = DS_READ_B128_gfx9 %addr, 0, 0, implicit $exec
+ %l2:vreg_128_align2 = DS_READ_B128_gfx9 %addr, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %v1, implicit %v2, implicit %l1, implicit %l2
+...
+
+---
+name: addr_multi_producer
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr20
+
+ ; CHECK-LABEL: name: addr_multi_producer
+ ; CHECK: liveins: $vgpr20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = GLOBAL_LOAD_DWORD killed renamable $vgpr2_vgpr3, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 2, killed $vgpr20, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: addr_multi_producer
+ ; CHECK-NO-ANTIHINT: liveins: $vgpr20
+ ; CHECK-NO-ANTIHINT-NEXT: {{ $}}
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = GLOBAL_LOAD_DWORD killed renamable $vgpr2_vgpr3, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAIT_XCNT 0
+ ; CHECK-NO-ANTIHINT-NEXT: S_WAITCNT_DEPCTR .VmVsrc_0
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 2, killed $vgpr20, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %s:vgpr_32 = COPY $vgpr20
+ %a0:vreg_64_align2 = IMPLICIT_DEF
+ %a1:vreg_64_align2 = IMPLICIT_DEF
+ %l0:vgpr_32 = GLOBAL_LOAD_DWORD %a0, 0, 0, implicit $exec
+ %l1:vgpr_32 = GLOBAL_LOAD_DWORD %a1, 0, 0, implicit $exec
+ %c1:vgpr_32 = V_ADD_U32_e32 1, %s, implicit $exec
+ %c2:vgpr_32 = V_ADD_U32_e32 2, %s, implicit $exec
+ S_ENDPGM 0, implicit %l0, implicit %l1, implicit %c1, implicit %c2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-multi-rule.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-multi-rule.gfx1250.mir
new file mode 100644
index 00000000000000..9974cfb2d47c2d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-multi-rule.gfx1250.mir
@@ -0,0 +1,351 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=wmma-war-ab,trans-war,mem-addr-war %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=CHECK-NO-ANTIHINT
+
+--- |
+ define amdgpu_kernel void @test_wmma_f32_16x16x32_bf16_anti_hints() #0 { ret void }
+ define amdgpu_kernel void @test_wmma_f32_16x16x64_fp8_fp8_anti_hints() #0 { ret void }
+ define amdgpu_kernel void @test_wmma_f32_16x16x128_fp8_fp8_anti_hints() #0 { ret void }
+ define amdgpu_kernel void @test_expf32() #0 { ret void }
+ define amdgpu_kernel void @test_expf32_dsread() #0 { ret void }
+ define amdgpu_kernel void @test_repeat_rate_clobber() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+---
+# Test: V_WMMA_F32_16X16X32_BF16 - Pattern "0EIIEEIIV" with 4 VALU slots
+name: test_wmma_f32_16x16x32_bf16_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_wmma_f32_16x16x32_bf16_anti_hints
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_wmma_f32_16x16x32_bf16_anti_hints
+ ; CHECK-NO-ANTIHINT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+---
+# Test: V_WMMA_F32_16X16X64_FP8_FP8 - Pattern "0EEIVV" with 1 VALU slot
+name: test_wmma_f32_16x16x64_fp8_fp8_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_wmma_f32_16x16x64_fp8_fp8_anti_hints
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr0
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_wmma_f32_16x16x64_fp8_fp8_anti_hints
+ ; CHECK-NO-ANTIHINT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, killed $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ S_ENDPGM 0,implicit %6, implicit %7, implicit %8
+...
+---
+# Test: V_WMMA_F32_16X16X128_FP8_FP8 - Pattern "0EEIEEIIVV" with 4 VALU slots
+name: test_wmma_f32_16x16x128_fp8_fp8_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_wmma_f32_16x16x128_fp8_fp8_anti_hints
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_wmma_f32_16x16x128_fp8_fp8_anti_hints
+ ; CHECK-NO-ANTIHINT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+# Test: V_EXP_F32_e32 - with 1 VALU slots
+name: test_expf32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_expf32
+ ; CHECK: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 4, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr7 = V_ADD_U32_e32 5, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 6, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 7, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 8, killed $vgpr4, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr5, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr4
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_expf32
+ ; CHECK-NO-ANTIHINT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr5 = V_ADD_U32_e32 3, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6 = V_ADD_U32_e32 4, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr7 = V_ADD_U32_e32 5, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8 = V_ADD_U32_e32 6, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr9 = V_ADD_U32_e32 7, $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = V_ADD_U32_e32 8, killed $vgpr4, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr4
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %0.sub1:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub0:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub1:vreg_64_align2 = nofpexcept V_EXP_F32_e32 %0.sub0, implicit $mode, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+# Test: V_EXP_F32_e32 - with 1 VALU slots
+name: test_expf32_dsread
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_expf32_dsread
+ ; CHECK: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr10 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = nofpexcept V_EXP_F32_e32 killed $vgpr8, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr10, 12608, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 2, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr7 = V_ADD_U32_e32 3, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 4, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 5, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr11 = V_ADD_U32_e32 6, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr12 = V_ADD_U32_e32 7, $vgpr10, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr10 = V_ADD_U32_e32 8, killed $vgpr10, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr11, implicit killed renamable $vgpr12, implicit killed renamable $vgpr10
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_expf32_dsread
+ ; CHECK-NO-ANTIHINT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr10 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: dead renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = V_ADD_U32_e32 1, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr5 = nofpexcept V_EXP_F32_e32 killed $vgpr8, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr10, 12608, 0, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6 = V_ADD_U32_e32 2, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr7 = V_ADD_U32_e32 3, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr8 = V_ADD_U32_e32 4, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr9 = V_ADD_U32_e32 5, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr11 = V_ADD_U32_e32 6, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr12 = V_ADD_U32_e32 7, $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr10 = V_ADD_U32_e32 8, killed $vgpr10, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr11, implicit killed renamable $vgpr12, implicit killed renamable $vgpr10
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %0.sub1:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub0:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub1:vreg_64_align2 = nofpexcept V_EXP_F32_e32 %0.sub0, implicit $mode, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4:vgpr_32, 12608, 0, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: test_repeat_rate_clobber
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: test_repeat_rate_clobber
+ ; CHECK: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr10 = V_ADD_U32_e32 1, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr11 = nofpexcept V_EXP_F32_e32 killed $vgpr8, implicit $mode, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr12_vgpr13 = nofpexcept V_CVT_SCALEF32_PK8_FP8_F32_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 1065353216, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr14, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr14, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10_vgpr11, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr12_vgpr13
+ ;
+ ; CHECK-NO-ANTIHINT-LABEL: name: test_repeat_rate_clobber
+ ; CHECK-NO-ANTIHINT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr12 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr10 = V_ADD_U32_e32 1, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr11 = nofpexcept V_EXP_F32_e32 killed $vgpr8, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: early-clobber renamable $vgpr8_vgpr9 = nofpexcept V_CVT_SCALEF32_PK8_FP8_F32_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 1065353216, implicit $mode, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr12, implicit $exec
+ ; CHECK-NO-ANTIHINT-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr10_vgpr11, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8_vgpr9
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %0.sub1:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub0:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub1:vreg_64_align2 = nofpexcept V_EXP_F32_e32 %0.sub0, implicit $mode, implicit $exec
+ %15:vreg_64_align2 = nofpexcept V_CVT_SCALEF32_PK8_FP8_F32_e64 %5, 1065353216, implicit $mode, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-swmmac-war-index.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-swmmac-war-index.gfx1250.mir
new file mode 100644
index 00000000000000..0c34833e712c48
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-swmmac-war-index.gfx1250.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=swmmac-war-index %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=NOAH
+
+--- |
+ define amdgpu_kernel void @war_swmmac_index_src2() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: war_swmmac_index_src2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_swmmac_index_src2
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33
+ ;
+ ; NOAH-LABEL: name: war_swmmac_index_src2
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr33
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-trans-src0.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-trans-src0.gfx1250.mir
new file mode 100644
index 00000000000000..edec99b26cb9d8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-trans-src0.gfx1250.mir
@@ -0,0 +1,166 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=trans-war %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=NOAH
+
+--- |
+ define amdgpu_kernel void @trans_src0_valu_def() #0 { ret void }
+ define amdgpu_kernel void @trans_war_wmma_dst_not_hinted() #0 { ret void }
+ define amdgpu_kernel void @trans_war_dpp_old_not_src0() #0 { ret void }
+ define amdgpu_kernel void @trans_coexec() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: trans_src0_valu_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins:
+
+ ; CHECK-LABEL: name: trans_src0_valu_def
+ ; CHECK: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 4, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr7 = V_ADD_U32_e32 5, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 6, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 7, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 8, killed $vgpr4, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr5, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr4
+ ;
+ ; NOAH-LABEL: name: trans_src0_valu_def
+ ; NOAH: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 3, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 4, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 5, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 6, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 7, $vgpr4, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 8, killed $vgpr4, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2_vgpr3, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr4
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %6:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %0.sub1:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub0:vreg_64_align2 = V_ADD_U32_e32 1, %4, implicit $exec
+ %6.sub1:vreg_64_align2 = nofpexcept V_EXP_F32_e32 %0.sub0, implicit $mode, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: trans_war_wmma_dst_not_hinted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: trans_war_wmma_dst_not_hinted
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 killed $vgpr16, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+ ;
+ ; NOAH-LABEL: name: trans_war_wmma_dst_not_hinted
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 killed $vgpr16, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vreg_256_align2 = IMPLICIT_DEF
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = nofpexcept V_EXP_F32_e32 %0, implicit $mode, implicit $exec
+ %4:vreg_256_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %2, %3, 8, %4, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %5
+...
+
+---
+name: trans_war_dpp_old_not_src0
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: trans_war_dpp_old_not_src0
+ ; CHECK: renamable $vgpr0 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead renamable $vgpr0 = V_EXP_F32_dpp killed $vgpr0, 0, $vgpr1, 1, 15, 15, 1, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: trans_war_dpp_old_not_src0
+ ; NOAH: renamable $vgpr0 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead renamable $vgpr0 = V_EXP_F32_dpp killed $vgpr0, 0, $vgpr1, 1, 15, 15, 1, implicit $mode, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr2, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr0
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ dead %3:vgpr_32 = V_EXP_F32_dpp %0, 0, %1, 1, 15, 15, 1, implicit $mode, implicit $exec
+ %4:vgpr_32 = V_ADD_U32_e32 1, %2, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %2, implicit %4
+...
+
+---
+name: trans_coexec
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0' }
+ - { reg: '$vgpr1' }
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+
+ ; CHECK-LABEL: name: trans_coexec
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr2 = V_SQRT_F32_e32 $vgpr0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+ ; CHECK-NEXT: dead renamable $vgpr0 = V_ADD_F32_e32 killed $vgpr2, $vgpr1, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr1
+ ;
+ ; NOAH-LABEL: name: trans_coexec
+ ; NOAH: liveins: $vgpr0, $vgpr1
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr2 = V_SQRT_F32_e32 $vgpr0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+ ; NOAH-NEXT: dead renamable $vgpr1 = V_ADD_F32_e32 killed $vgpr2, $vgpr0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_SQRT_F32_e32 %0, implicit $mode, implicit $exec
+ %3:vgpr_32 = V_ADD_F32_e32 %0, %1, implicit $mode, implicit $exec
+ %4:vgpr_32 = V_ADD_F32_e32 %2, %3, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-va-vdst.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-va-vdst.gfx1250.mir
new file mode 100644
index 00000000000000..0c6d55a60570de
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-va-vdst.gfx1250.mir
@@ -0,0 +1,137 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-for-va-vdst=true -amdgpu-anti-hints-rules=va-vdst-war %s -o - | FileCheck %s --check-prefix=CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -amdgpu-expert-scheduling-mode=true -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec,si-insert-waitcnts -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefix=NOAH
+
+--- |
+ define amdgpu_kernel void @test_valu_dsread_anti_hints() #0 {
+ bb.0:
+ ret void
+ }
+
+ define amdgpu_kernel void @test_valu_buffer_load_anti_hints() #0 {
+ bb.0:
+ ret void
+ }
+
+ define amdgpu_kernel void @test_pk_add_dsread_anti_hints() #0 {
+ bb.0:
+ ret void
+ }
+
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+
+...
+---
+name: test_valu_dsread_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_valu_dsread_anti_hints
+ ; CHECK: liveins: $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr4 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 1, $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 2, killed $vgpr4, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr5, implicit killed renamable $vgpr4
+ ;
+ ; NOAH-LABEL: name: test_valu_dsread_anti_hints
+ ; NOAH: liveins: $vgpr10
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 2, killed $vgpr0, implicit $exec
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5
+ liveins:$vgpr10
+
+ %0:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = COPY $vgpr10
+ %1:vgpr_32 = V_ADD_U32_e32 1, %0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e32 2, %0, implicit $exec
+ %3:vreg_128_align2 = DS_READ_B128_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %3, implicit %1, implicit %2
+...
+---
+name: test_valu_buffer_load_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4
+
+ ; CHECK-LABEL: name: test_valu_buffer_load_anti_hints
+ ; CHECK: liveins: $sgpr4, $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, killed $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = BUFFER_LOAD_DWORD_OFFEN killed renamable $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr1, implicit killed renamable $vgpr0
+ ;
+ ; NOAH-LABEL: name: test_valu_buffer_load_anti_hints
+ ; NOAH: liveins: $sgpr4, $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, killed $vgpr0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = BUFFER_LOAD_DWORD_OFFEN killed renamable $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr2, implicit killed renamable $vgpr1, implicit killed renamable $vgpr0
+ %0:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = COPY $vgpr10
+ %1:vgpr_32 = V_ADD_U32_e32 1, %0, implicit $exec
+ %2:vgpr_32 = V_ADD_U32_e32 2, %0, implicit $exec
+ %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %3, implicit %1, implicit %2
+...
+---
+name: test_pk_add_dsread_anti_hints
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr10
+
+ ; CHECK-LABEL: name: test_pk_add_dsread_anti_hints
+ ; CHECK: liveins: $vgpr10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; CHECK-NEXT: S_WAIT_KMCNT 0
+ ; CHECK-NEXT: renamable $vgpr4_vgpr5 = V_PK_ADD_F32 8, killed $vgpr4_vgpr5, 8, killed $vgpr6_vgpr7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+ ;
+ ; NOAH-LABEL: name: test_pk_add_dsread_anti_hints
+ ; NOAH: liveins: $vgpr10
+ ; NOAH-NEXT: {{ $}}
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr2_vgpr3 = IMPLICIT_DEF
+ ; NOAH-NEXT: S_SETREG_IMM32_B32 2, 2074, implicit-def $mode, implicit $mode
+ ; NOAH-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; NOAH-NEXT: S_WAIT_KMCNT 0
+ ; NOAH-NEXT: renamable $vgpr4_vgpr5 = V_PK_ADD_F32 8, killed $vgpr0_vgpr1, 8, killed $vgpr2_vgpr3, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_WAITCNT_DEPCTR .VaVdst_0
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr4_vgpr5, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = COPY $vgpr10
+ %3:vreg_64_align2 = V_PK_ADD_F32 8, %0, 8, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_128_align2 = DS_READ_B128_gfx9 %2:vgpr_32, 0, 0, implicit $exec
+ S_ENDPGM 0, implicit %3, implicit %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-war.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-war.gfx1250.mir
new file mode 100644
index 00000000000000..b5e98f8daa5a46
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-war.gfx1250.mir
@@ -0,0 +1,1610 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=wmma-war-ab %s -o - | FileCheck %s --check-prefixes=SAME,CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefixes=SAME,NOAH
+
+--- |
+ define amdgpu_kernel void @war_valu_wmma_bf16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_f16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_f16_dst_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_fp8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_iu8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_fp8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_f8f6f4_f8_f8() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_f8f6f4_f4_f4() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_scale_f8f6f4() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_32x16x128_f4() #0 { ret void }
+ define amdgpu_kernel void @war_valu_swmmac_bf16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_valu_swmmac_f16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_valu_swmmac_f16_dst_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_valu_swmmac_iu8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @war_valu_wmma_f32_16x16x4_f32() #0 { ret void }
+ define amdgpu_kernel void @war_src0_only_dead() #0 { ret void }
+ define amdgpu_kernel void @war_src1_only_dead() #0 { ret void }
+ define amdgpu_kernel void @war_tied_dst_only_dead() #0 { ret void }
+ define amdgpu_kernel void @war_trans_def() #0 { ret void }
+ define amdgpu_kernel void @war_copy_def() #0 { ret void }
+ define amdgpu_kernel void @war_ds_def_only() #0 { ret void }
+ define amdgpu_kernel void @war_ds_def_with_valu() #0 { ret void }
+ define amdgpu_kernel void @war_ds_def_countmask() #0 { ret void }
+ define amdgpu_kernel void @war_vmem_def() #0 { ret void }
+ define amdgpu_kernel void @war_countmask_salu_between() #0 { ret void }
+ define amdgpu_kernel void @war_window_expires_fp8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @war_window_expires() #0 { ret void }
+ define amdgpu_kernel void @war_second_wmma_flushes_family() #0 { ret void }
+ define amdgpu_kernel void @war_non_xdl_wmma_flushes_family() #0 { ret void }
+ define amdgpu_kernel void @war_cross_block_no_hints() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: war_valu_wmma_bf16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_bf16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_bf16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_f16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_f16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_f16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_f16_dst_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_f16_dst_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = V_ADD_U32_e32 2, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr23 = V_ADD_U32_e32 3, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 4, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr20, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr24, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_f16_dst_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr20, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_WMMA_F16_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_fp8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_fp8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_fp8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_iu8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_iu8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr29 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 8, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 9, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 10, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 11, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 12, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 13, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 14, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 15, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr7 = V_ADD_U32_e32 16, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_iu8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 9, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 10, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 11, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 12, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 13, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr13 = V_ADD_U32_e32 14, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr14 = V_ADD_U32_e32 15, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr15 = V_ADD_U32_e32 16, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr12, implicit killed renamable $vgpr13, implicit killed renamable $vgpr14, implicit killed renamable $vgpr15
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, %0, 8, %1, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ %15:vgpr_32 = V_ADD_U32_e32 9, %4, implicit $exec
+ %16:vgpr_32 = V_ADD_U32_e32 10, %4, implicit $exec
+ %17:vgpr_32 = V_ADD_U32_e32 11, %4, implicit $exec
+ %18:vgpr_32 = V_ADD_U32_e32 12, %4, implicit $exec
+ %19:vgpr_32 = V_ADD_U32_e32 13, %4, implicit $exec
+ %20:vgpr_32 = V_ADD_U32_e32 14, %4, implicit $exec
+ %21:vgpr_32 = V_ADD_U32_e32 15, %4, implicit $exec
+ %22:vgpr_32 = V_ADD_U32_e32 16, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %19, implicit %20, implicit %21, implicit %22
+...
+
+---
+name: war_valu_wmma_fp8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_fp8_16x16x128
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_fp8_16x16x128
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_f8f6f4_f8_f8
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_f8f6f4_f8_f8
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_f8f6f4_f8_f8
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_f8f6f4_f4_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_f8f6f4_f4_f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr4 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_f8f6f4_f4_f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr %0, %1, 8, %5, 4, 4, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_scale_f8f6f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_scale_f8f6f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, killed $vgpr40, killed $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_ADD_U32_e32 1, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_scale_f8f6f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, killed $vgpr40, killed $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr42, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32_lo256 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 0, %5, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_32x16x128_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_wmma_32x16x128_f4
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_32X16X128_F4_w32_twoaddr killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: war_valu_wmma_32x16x128_f4
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_32X16X128_F4_w32_twoaddr killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = V_WMMA_F32_32X16X128_F4_w32_twoaddr %0, %1, 8, %5, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_swmmac_bf16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_swmmac_bf16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ ;
+ ; NOAH-LABEL: name: war_valu_swmmac_bf16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_valu_swmmac_f16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_swmmac_f16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ ;
+ ; NOAH-LABEL: name: war_valu_swmmac_f16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_valu_swmmac_f16_dst_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_swmmac_f16_dst_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, killed $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 1, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 2, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 3, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 4, killed $vgpr29, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr30, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ ;
+ ; NOAH-LABEL: name: war_valu_swmmac_f16_dst_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, killed $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, killed $vgpr29, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_valu_swmmac_iu8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_valu_swmmac_iu8_16x16x128
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 3, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr36 = V_ADD_U32_e32 4, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr34, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_valu_swmmac_iu8_16x16x128
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, killed $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr34, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_valu_wmma_f32_16x16x4_f32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: war_valu_wmma_f32_16x16x4_f32
+ ; SAME: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr12 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X4_F32_w32_twoaddr killed $vgpr8_vgpr9, killed $vgpr10_vgpr11, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr10 = V_ADD_U32_e32 3, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr11 = V_ADD_U32_e32 4, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr13 = V_ADD_U32_e32 5, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr14 = V_ADD_U32_e32 6, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr15 = V_ADD_U32_e32 7, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr12 = V_ADD_U32_e32 8, killed $vgpr12, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr13, implicit killed renamable $vgpr14, implicit killed renamable $vgpr15, implicit killed renamable $vgpr12
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X4_F32_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_src0_only_dead
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_src0_only_dead
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_src0_only_dead
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %1, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_src1_only_dead
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_src1_only_dead
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr10 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr11 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11
+ ;
+ ; NOAH-LABEL: name: war_src1_only_dead
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr13 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr14 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr15 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr12, implicit killed renamable $vgpr13, implicit killed renamable $vgpr14, implicit killed renamable $vgpr15
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_tied_dst_only_dead
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: war_tied_dst_only_dead
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_trans_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_trans_def
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_trans_def
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %8:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %9:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %10:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %11:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %12:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %13:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %14:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_copy_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_copy_def
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = COPY renamable $vgpr24
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr9 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9
+ ;
+ ; NOAH-LABEL: name: war_copy_def
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = COPY renamable $vgpr24
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr12 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr12
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %20:vreg_256_align2 = COPY %0
+ %21:vgpr_32 = COPY %4
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %20, implicit %21, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_ds_def_only
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_ds_def_only
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr36 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr36, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 renamable $vgpr36, 16, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35 = DS_READ_B128_gfx9 renamable $vgpr36, 32, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr36_vgpr37_vgpr38_vgpr39 = DS_READ_B128_gfx9 killed renamable $vgpr36, 48, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39
+ ;
+ ; NOAH-LABEL: name: war_ds_def_only
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr24, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = DS_READ_B128_gfx9 renamable $vgpr24, 16, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = DS_READ_B128_gfx9 renamable $vgpr24, 32, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = DS_READ_B128_gfx9 killed renamable $vgpr24, 48, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec
+ %8:vreg_128_align2 = DS_READ_B128_gfx9 %4, 16, 0, implicit $exec
+ %9:vreg_128_align2 = DS_READ_B128_gfx9 %4, 32, 0, implicit $exec
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %4, 48, 0, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_ds_def_with_valu
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_ds_def_with_valu
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr28, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr29 = V_ADD_U32_e32 2, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 3, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = V_ADD_U32_e32 4, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 5, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 6, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 7, $vgpr28, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 8, killed $vgpr28, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2
+ ;
+ ; NOAH-LABEL: name: war_ds_def_with_valu
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr24, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_ds_def_countmask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_ds_def_countmask
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr44 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr44, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 renamable $vgpr44, 16, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35 = DS_READ_B128_gfx9 renamable $vgpr44, 32, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr36_vgpr37_vgpr38_vgpr39 = DS_READ_B128_gfx9 renamable $vgpr44, 48, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43 = DS_READ_B128_gfx9 renamable $vgpr44, 64, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44_vgpr45_vgpr46_vgpr47 = DS_READ_B128_gfx9 killed renamable $vgpr44, 80, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35, implicit killed renamable $vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr40_vgpr41_vgpr42_vgpr43, implicit killed renamable $vgpr44_vgpr45_vgpr46_vgpr47
+ ;
+ ; NOAH-LABEL: name: war_ds_def_countmask
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = DS_READ_B128_gfx9 renamable $vgpr28, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = DS_READ_B128_gfx9 renamable $vgpr28, 16, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11 = DS_READ_B128_gfx9 renamable $vgpr28, 32, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr12_vgpr13_vgpr14_vgpr15 = DS_READ_B128_gfx9 renamable $vgpr28, 48, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr28, 64, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 killed renamable $vgpr28, 80, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11, implicit killed renamable $vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr28_vgpr29_vgpr30_vgpr31
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec
+ %8:vreg_128_align2 = DS_READ_B128_gfx9 %4, 16, 0, implicit $exec
+ %9:vreg_128_align2 = DS_READ_B128_gfx9 %4, 32, 0, implicit $exec
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %4, 48, 0, implicit $exec
+ %11:vreg_128_align2 = DS_READ_B128_gfx9 %4, 64, 0, implicit $exec
+ %12:vreg_128_align2 = DS_READ_B128_gfx9 %4, 80, 0, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12
+...
+
+---
+name: war_vmem_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_vmem_def
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 4, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 8, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = GLOBAL_LOAD_DWORD killed renamable $vgpr24_vgpr25, 12, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr24
+ ;
+ ; NOAH-LABEL: name: war_vmem_def
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 4, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 8, 0, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = GLOBAL_LOAD_DWORD killed renamable $vgpr24_vgpr25, 12, 0, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vreg_64_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = GLOBAL_LOAD_DWORD %4, 0, 0, implicit $exec
+ %8:vgpr_32 = GLOBAL_LOAD_DWORD %4, 4, 0, implicit $exec
+ %9:vgpr_32 = GLOBAL_LOAD_DWORD %4, 8, 0, implicit $exec
+ %10:vgpr_32 = GLOBAL_LOAD_DWORD %4, 12, 0, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_countmask_salu_between
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_countmask_salu_between
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1
+ ; CHECK-NEXT: renamable $sgpr1 = S_MOV_B32 2
+ ; CHECK-NEXT: renamable $sgpr2 = S_MOV_B32 3
+ ; CHECK-NEXT: renamable $sgpr3 = S_MOV_B32 4
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $sgpr0, implicit killed renamable $sgpr1, implicit killed renamable $sgpr2, implicit killed renamable $sgpr3, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ ;
+ ; NOAH-LABEL: name: war_countmask_salu_between
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $sgpr0 = S_MOV_B32 1
+ ; NOAH-NEXT: renamable $sgpr1 = S_MOV_B32 2
+ ; NOAH-NEXT: renamable $sgpr2 = S_MOV_B32 3
+ ; NOAH-NEXT: renamable $sgpr3 = S_MOV_B32 4
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $sgpr0, implicit killed renamable $sgpr1, implicit killed renamable $sgpr2, implicit killed renamable $sgpr3, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %15:sgpr_32 = S_MOV_B32 1
+ %16:sgpr_32 = S_MOV_B32 2
+ %17:sgpr_32 = S_MOV_B32 3
+ %18:sgpr_32 = S_MOV_B32 4
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %15, implicit %16, implicit %17, implicit %18, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: war_window_expires_fp8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_window_expires_fp8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 2, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr0
+ ;
+ ; NOAH-LABEL: name: war_window_expires_fp8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8
+...
+
+---
+name: war_window_expires
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_window_expires
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 5, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr0
+ ;
+ ; NOAH-LABEL: name: war_window_expires
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11
+...
+
+---
+name: war_second_wmma_flushes_family
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_second_wmma_flushes_family
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr48 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_ADD_U32_e32 1, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 2, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 3, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 4, killed $vgpr48, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr8
+ ;
+ ; NOAH-LABEL: name: war_second_wmma_flushes_family
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr48 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 3, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 4, killed $vgpr48, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %15:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %16:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %2, %3, 8, %15, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3, implicit %6, implicit %16, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_non_xdl_wmma_flushes_family
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: war_non_xdl_wmma_flushes_family
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr34_vgpr35 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr36 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X4_F32_w32_twoaddr killed $vgpr32_vgpr33, killed $vgpr34_vgpr35, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr8 = V_ADD_U32_e32 4, killed $vgpr36, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr8
+ ;
+ ; NOAH-LABEL: name: war_non_xdl_wmma_flushes_family
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr34_vgpr35 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr36 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X4_F32_w32_twoaddr killed $vgpr32_vgpr33, killed $vgpr34_vgpr35, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr8 = V_ADD_U32_e32 1, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr9 = V_ADD_U32_e32 2, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr10 = V_ADD_U32_e32 3, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr11 = V_ADD_U32_e32 4, killed $vgpr36, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %3:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %15:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %16:vreg_256_align2 = V_WMMA_F32_16X16X4_F32_w32_twoaddr %2, %3, 8, %15, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %16, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: war_cross_block_no_hints
+tracksRegLiveness: true
+body: |
+ ; SAME-LABEL: name: war_cross_block_no_hints
+ ; SAME: bb.0:
+ ; SAME-NEXT: successors: %bb.1(0x80000000)
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, killed $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: S_BRANCH %bb.1
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: bb.1:
+ ; SAME-NEXT: liveins: $vgpr8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: renamable $vgpr9 = V_ADD_U32_e32 1, $vgpr8, implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: renamable $vgpr10 = V_ADD_U32_e32 2, $vgpr8, implicit $exec
+ ; SAME-NEXT: renamable $vgpr11 = V_ADD_U32_e32 3, $vgpr8, implicit $exec
+ ; SAME-NEXT: renamable $vgpr8 = V_ADD_U32_e32 4, killed $vgpr8, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr9, implicit killed renamable $vgpr10, implicit killed renamable $vgpr11, implicit killed renamable $vgpr8
+ bb.0:
+ successors: %bb.1
+
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-waw.gfx1250.mir b/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-waw.gfx1250.mir
new file mode 100644
index 00000000000000..8578d6edc763a2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/anti-hints-wmma-waw.gfx1250.mir
@@ -0,0 +1,2293 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=wmma-waw %s -o - | FileCheck %s --check-prefixes=SAME,CHECK
+# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=amdgpu-pre-ra-optimizations,greedy,machineverifier,virtregrewriter,post-RA-hazard-rec -amdgpu-anti-hints-rules=none %s -o - | FileCheck %s --check-prefixes=SAME,NOAH
+
+
+--- |
+ define amdgpu_kernel void @waw_valu_wmma_bf16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_f16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_f16_dst_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_fp8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_iu8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_fp8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_f8f6f4_f8_f8() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_f8f6f4_f4_f4() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_scale_f8f6f4() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_32x16x128_f4() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_swmmac_bf16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_swmmac_f16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_swmmac_f16_dst_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_swmmac_iu8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @waw_valu_wmma_f32_16x16x4_f32() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_bf16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_f16_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_f16_dst_16x16x32() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_fp8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_iu8_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_fp8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_f8f6f4_f8_f8() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_f8f6f4_f4_f4() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_scale_f8f6f4() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_32x16x128_f4() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_swmmac_bf16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_swmmac_f16_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_swmmac_f16_dst_16x16x64() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_swmmac_iu8_16x16x128() #0 { ret void }
+ define amdgpu_kernel void @waw_trans_wmma_f32_16x16x4_f32() #0 { ret void }
+ define amdgpu_kernel void @waw_dst_only_dead() #0 { ret void }
+ define amdgpu_kernel void @waw_dst_live_gated() #0 { ret void }
+ define amdgpu_kernel void @waw_dst_and_srcs_dead() #0 { ret void }
+ define amdgpu_kernel void @waw_swmmac_dst_index_live() #0 { ret void }
+ define amdgpu_kernel void @waw_mixed_valu_trans() #0 { ret void }
+ define amdgpu_kernel void @waw_copy_def() #0 { ret void }
+ define amdgpu_kernel void @waw_wide_valu_def() #0 { ret void }
+ define amdgpu_kernel void @waw_ds_def_not_hinted() #0 { ret void }
+ define amdgpu_kernel void @waw_vmem_def_not_hinted() #0 { ret void }
+ define amdgpu_kernel void @waw_countmask_salu_between() #0 { ret void }
+ define amdgpu_kernel void @waw_window_expires() #0 { ret void }
+ define amdgpu_kernel void @waw_second_wmma_flushes_family() #0 { ret void }
+ define amdgpu_kernel void @waw_non_xdl_wmma_flushes_family() #0 { ret void }
+ define amdgpu_kernel void @waw_cross_block_no_hints() #0 { ret void }
+
+ attributes #0 = {nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"}
+...
+
+---
+name: waw_valu_wmma_bf16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_bf16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_bf16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_f16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_f16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_f16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_f16_dst_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_f16_dst_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = V_ADD_U32_e32 2, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr23 = V_ADD_U32_e32 3, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 4, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr20, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr20, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_f16_dst_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 5, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 6, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 7, $vgpr20, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 8, killed $vgpr20, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr20
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_WMMA_F16_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_fp8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_fp8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_fp8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_iu8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_iu8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr29 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 8, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 9, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 10, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 11, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 12, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = V_ADD_U32_e32 13, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = V_ADD_U32_e32 14, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = V_ADD_U32_e32 15, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr23 = V_ADD_U32_e32 16, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_iu8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 9, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 10, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 11, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr28 = V_ADD_U32_e32 12, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr29 = V_ADD_U32_e32 13, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr30 = V_ADD_U32_e32 14, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr31 = V_ADD_U32_e32 15, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 16, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr24
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, %0, 8, %1, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ %15:vgpr_32 = V_ADD_U32_e32 9, %4, implicit $exec
+ %16:vgpr_32 = V_ADD_U32_e32 10, %4, implicit $exec
+ %17:vgpr_32 = V_ADD_U32_e32 11, %4, implicit $exec
+ %18:vgpr_32 = V_ADD_U32_e32 12, %4, implicit $exec
+ %19:vgpr_32 = V_ADD_U32_e32 13, %4, implicit $exec
+ %20:vgpr_32 = V_ADD_U32_e32 14, %4, implicit $exec
+ %21:vgpr_32 = V_ADD_U32_e32 15, %4, implicit $exec
+ %22:vgpr_32 = V_ADD_U32_e32 16, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %19, implicit %20, implicit %21, implicit %22
+...
+
+---
+name: waw_valu_wmma_fp8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_fp8_16x16x128
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_fp8_16x16x128
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_f8f6f4_f8_f8
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_f8f6f4_f8_f8
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_f8f6f4_f8_f8
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_f8f6f4_f4_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_f8f6f4_f4_f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_f8f6f4_f4_f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr %0, %1, 8, %5, 4, 4, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_scale_f8f6f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_scale_f8f6f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 1, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 2, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr45 = V_ADD_U32_e32 3, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr46 = V_ADD_U32_e32 4, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = V_ADD_U32_e32 5, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = V_ADD_U32_e32 6, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 7, $vgpr42, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 8, killed $vgpr42, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr45, implicit killed renamable $vgpr46, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_scale_f8f6f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 3, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = V_ADD_U32_e32 4, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = V_ADD_U32_e32 5, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = V_ADD_U32_e32 6, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = V_ADD_U32_e32 7, $vgpr42, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = V_ADD_U32_e32 8, killed $vgpr42, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32_lo256 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 0, %5, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_32x16x128_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_wmma_32x16x128_f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_valu_wmma_32x16x128_f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr40, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr40, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = V_WMMA_F32_32X16X128_F4_w32_twoaddr %0, %1, 8, %5, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_swmmac_bf16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_swmmac_bf16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_valu_swmmac_bf16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_valu_swmmac_f16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_swmmac_f16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_valu_swmmac_f16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_valu_swmmac_f16_dst_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_swmmac_f16_dst_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = V_ADD_U32_e32 1, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = V_ADD_U32_e32 2, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 3, $vgpr29, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 4, killed $vgpr29, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr28, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_valu_swmmac_f16_dst_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr29, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 4, killed $vgpr29, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr28, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_valu_swmmac_iu8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_valu_swmmac_iu8_16x16x128
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 1, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr36 = V_ADD_U32_e32 2, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr37 = V_ADD_U32_e32 3, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38 = V_ADD_U32_e32 4, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 5, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 6, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 7, $vgpr34, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 8, killed $vgpr34, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ ;
+ ; NOAH-LABEL: name: waw_valu_swmmac_iu8_16x16x128
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 4, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr28 = V_ADD_U32_e32 5, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr29 = V_ADD_U32_e32 6, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr30 = V_ADD_U32_e32 7, $vgpr34, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr31 = V_ADD_U32_e32 8, killed $vgpr34, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_valu_wmma_f32_16x16x4_f32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_valu_wmma_f32_16x16x4_f32
+ ; SAME: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr12 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr8_vgpr9, $vgpr10_vgpr11, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr4 = V_ADD_U32_e32 5, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr5 = V_ADD_U32_e32 6, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr6 = V_ADD_U32_e32 7, $vgpr12, implicit $exec
+ ; SAME-NEXT: renamable $vgpr7 = V_ADD_U32_e32 8, killed $vgpr12, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr10_vgpr11, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X4_F32_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_trans_wmma_bf16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_bf16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_bf16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_f16_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_f16_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_f16_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_f16_dst_16x16x32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_f16_dst_16x16x32
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 killed $vgpr20, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr24, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_f16_dst_16x16x32
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr20 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19 = V_WMMA_F16_16X16X32_F16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 killed $vgpr20, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr20
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_WMMA_F16_16X16X32_F16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_fp8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_fp8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_fp8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_iu8_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_iu8_16x16x64
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr29 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_iu8_16x16x64
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr29 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr30 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr31 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr24
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_I32_16X16X64_IU8_w32_twoaddr 8, %0, 8, %1, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %108:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %109:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %110:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %111:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %112:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %113:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %114:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %115:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107, implicit %108, implicit %109, implicit %110, implicit %111, implicit %112, implicit %113, implicit %114, implicit %115
+...
+
+---
+name: waw_trans_wmma_fp8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_fp8_16x16x128
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_fp8_16x16x128
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_f8f6f4_f8_f8
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_f8f6f4_f8_f8
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_f8f6f4_f8_f8
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_f8f6f4_f4_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_f8f6f4_f4_f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_f8f6f4_f4_f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 4, 4, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X128_F8F6F4_f4_f4_w32_twoaddr %0, %1, 8, %5, 4, 4, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_scale_f8f6f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_scale_f8f6f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr45 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr46 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 killed $vgpr42, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr45, implicit killed renamable $vgpr46, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_scale_f8f6f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr41 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr42 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, $vgpr40, $vgpr41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr36 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr37 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr38 = nofpexcept V_EXP_F32_e32 $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr39 = nofpexcept V_EXP_F32_e32 killed $vgpr42, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32_lo256 = IMPLICIT_DEF
+ %3:vgpr_32_lo256 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_twoaddr %0, %1, 0, %5, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_32x16x128_f4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_wmma_32x16x128_f4
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr44 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43, implicit killed renamable $vgpr44, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_trans_wmma_32x16x128_f4
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_WMMA_F32_32X16X128_F4_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = nofpexcept V_EXP_F32_e32 $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = nofpexcept V_EXP_F32_e32 killed $vgpr40, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_512_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_512_align2 = IMPLICIT_DEF
+ %6:vreg_512_align2 = V_WMMA_F32_32X16X128_F4_w32_twoaddr %0, %1, 8, %5, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_swmmac_bf16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_swmmac_bf16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 killed $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_trans_swmmac_bf16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 killed $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %100, implicit %101, implicit %102, implicit %103
+...
+
+---
+name: waw_trans_swmmac_f16_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_swmmac_f16_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 killed $vgpr33, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_trans_swmmac_f16_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 killed $vgpr33, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %100, implicit %101, implicit %102, implicit %103
+...
+
+---
+name: waw_trans_swmmac_f16_dst_16x16x64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_swmmac_f16_dst_16x16x64
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr31 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 killed $vgpr29, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr28, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_trans_swmmac_f16_dst_16x16x64
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr29 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr29, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 killed $vgpr29, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr28, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_128_align2 = IMPLICIT_DEF
+ %6:vreg_128_align2 = V_SWMMAC_F16_16X16X64_F16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %100, implicit %101, implicit %102, implicit %103
+...
+
+---
+name: waw_trans_swmmac_iu8_16x16x128
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_trans_swmmac_iu8_16x16x128
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr36 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr37 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr38 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 killed $vgpr34, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr35, implicit killed renamable $vgpr36, implicit killed renamable $vgpr37, implicit killed renamable $vgpr38, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ ;
+ ; NOAH-LABEL: name: waw_trans_swmmac_iu8_16x16x128
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr34 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr28 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr29 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr30 = nofpexcept V_EXP_F32_e32 $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr31 = nofpexcept V_EXP_F32_e32 killed $vgpr34, implicit $mode, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr29, implicit killed renamable $vgpr30, implicit killed renamable $vgpr31
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_I32_16X16X128_IU8_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_trans_wmma_f32_16x16x4_f32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_trans_wmma_f32_16x16x4_f32
+ ; SAME: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr12 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr8_vgpr9, $vgpr10_vgpr11, 8, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr0 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr1 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr2 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr3 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr4 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr5 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr6 = nofpexcept V_EXP_F32_e32 $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: renamable $vgpr7 = nofpexcept V_EXP_F32_e32 killed $vgpr12, implicit $mode, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr8_vgpr9, implicit killed renamable $vgpr10_vgpr11, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr6, implicit killed renamable $vgpr7
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X4_F32_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %100:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %101:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %102:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %103:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %104:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %105:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %106:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %107:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %100, implicit %101, implicit %102, implicit %103, implicit %104, implicit %105, implicit %106, implicit %107
+...
+
+---
+name: waw_dst_only_dead
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_dst_only_dead
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr24
+ ;
+ ; NOAH-LABEL: name: waw_dst_only_dead
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_dst_live_gated
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_dst_live_gated
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_dst_and_srcs_dead
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_dst_and_srcs_dead
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, killed $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr1 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr2 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr3 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0, implicit killed renamable $vgpr1, implicit killed renamable $vgpr2, implicit killed renamable $vgpr3
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_swmmac_dst_index_live
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_swmmac_dst_index_live
+ ; CHECK: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr34 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr35 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_swmmac_dst_index_live
+ ; NOAH: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 8, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32, 0, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr33, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr27 = V_ADD_U32_e32 4, killed $vgpr33, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_512_align2 = IMPLICIT_DEF
+ %2:vgpr_32 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_SWMMAC_F32_16X16X64_BF16_w32_twoaddr 8, %0, 8, %1, %5, %2, 0, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_mixed_valu_trans
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_mixed_valu_trans
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17
+ ;
+ ; NOAH-LABEL: name: waw_mixed_valu_trans
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = nofpexcept V_EXP_F32_e32 $vgpr24, implicit $mode, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %8:vgpr_32 = nofpexcept V_EXP_F32_e32 %4, implicit $mode, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12
+...
+
+---
+name: waw_copy_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_copy_def
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr25 = COPY renamable $vgpr24
+ ; SAME-NEXT: renamable $vgpr26 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr27 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr28 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; SAME-NEXT: renamable $vgpr24 = V_ADD_U32_e32 4, killed $vgpr24, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr24
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %20:vreg_256_align2 = COPY %6
+ %21:vgpr_32 = COPY %4
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %20, implicit %21, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_wide_valu_def
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_wide_valu_def
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26_vgpr27 = V_LSHLREV_B64_e64 1, $vgpr24_vgpr25, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28_vgpr29 = V_LSHLREV_B64_e64 2, $vgpr24_vgpr25, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr30_vgpr31 = V_LSHLREV_B64_e64 3, $vgpr24_vgpr25, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25 = V_LSHLREV_B64_e64 4, killed $vgpr24_vgpr25, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr26_vgpr27, implicit killed renamable $vgpr28_vgpr29, implicit killed renamable $vgpr30_vgpr31, implicit killed renamable $vgpr24_vgpr25
+ ;
+ ; NOAH-LABEL: name: waw_wide_valu_def
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17 = V_LSHLREV_B64_e64 1, $vgpr24_vgpr25, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18_vgpr19 = V_LSHLREV_B64_e64 2, $vgpr24_vgpr25, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20_vgpr21 = V_LSHLREV_B64_e64 3, $vgpr24_vgpr25, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22_vgpr23 = V_LSHLREV_B64_e64 4, killed $vgpr24_vgpr25, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17, implicit killed renamable $vgpr18_vgpr19, implicit killed renamable $vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vreg_64_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vreg_64_align2 = V_LSHLREV_B64_e64 1, %4, implicit $exec
+ %8:vreg_64_align2 = V_LSHLREV_B64_e64 2, %4, implicit $exec
+ %9:vreg_64_align2 = V_LSHLREV_B64_e64 3, %4, implicit $exec
+ %10:vreg_64_align2 = V_LSHLREV_B64_e64 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_ds_def_not_hinted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_ds_def_not_hinted
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr28 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19 = DS_READ_B128_gfx9 renamable $vgpr28, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr20_vgpr21_vgpr22_vgpr23 = DS_READ_B128_gfx9 renamable $vgpr28, 16, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27 = DS_READ_B128_gfx9 renamable $vgpr28, 32, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr28_vgpr29_vgpr30_vgpr31 = DS_READ_B128_gfx9 killed renamable $vgpr28, 48, 0, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19, implicit killed renamable $vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27, implicit killed renamable $vgpr28_vgpr29_vgpr30_vgpr31
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vreg_128_align2 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec
+ %8:vreg_128_align2 = DS_READ_B128_gfx9 %4, 16, 0, implicit $exec
+ %9:vreg_128_align2 = DS_READ_B128_gfx9 %4, 32, 0, implicit $exec
+ %10:vreg_128_align2 = DS_READ_B128_gfx9 %4, 48, 0, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_vmem_def_not_hinted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; SAME-LABEL: name: waw_vmem_def_not_hinted
+ ; SAME: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr24_vgpr25 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr16 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 0, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr17 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 4, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr18 = GLOBAL_LOAD_DWORD renamable $vgpr24_vgpr25, 8, 0, implicit $exec
+ ; SAME-NEXT: renamable $vgpr19 = GLOBAL_LOAD_DWORD killed renamable $vgpr24_vgpr25, 12, 0, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vreg_64_align2 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = GLOBAL_LOAD_DWORD %4, 0, 0, implicit $exec
+ %8:vgpr_32 = GLOBAL_LOAD_DWORD %4, 4, 0, implicit $exec
+ %9:vgpr_32 = GLOBAL_LOAD_DWORD %4, 8, 0, implicit $exec
+ %10:vgpr_32 = GLOBAL_LOAD_DWORD %4, 12, 0, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_countmask_salu_between
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_countmask_salu_between
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1
+ ; CHECK-NEXT: renamable $sgpr1 = S_MOV_B32 2
+ ; CHECK-NEXT: renamable $sgpr2 = S_MOV_B32 3
+ ; CHECK-NEXT: renamable $sgpr3 = S_MOV_B32 4
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr17 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr18 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr19 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0, implicit killed renamable $sgpr1, implicit killed renamable $sgpr2, implicit killed renamable $sgpr3, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ ;
+ ; NOAH-LABEL: name: waw_countmask_salu_between
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: renamable $sgpr0 = S_MOV_B32 1
+ ; NOAH-NEXT: renamable $sgpr1 = S_MOV_B32 2
+ ; NOAH-NEXT: renamable $sgpr2 = S_MOV_B32 3
+ ; NOAH-NEXT: renamable $sgpr3 = S_MOV_B32 4
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr21 = V_ADD_U32_e32 6, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr22 = V_ADD_U32_e32 7, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr23 = V_ADD_U32_e32 8, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $sgpr0, implicit killed renamable $sgpr1, implicit killed renamable $sgpr2, implicit killed renamable $sgpr3, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20, implicit killed renamable $vgpr21, implicit killed renamable $vgpr22, implicit killed renamable $vgpr23
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %15:sgpr_32 = S_MOV_B32 1
+ %16:sgpr_32 = S_MOV_B32 2
+ %17:sgpr_32 = S_MOV_B32 3
+ %18:sgpr_32 = S_MOV_B32 4
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ %12:vgpr_32 = V_ADD_U32_e32 6, %4, implicit $exec
+ %13:vgpr_32 = V_ADD_U32_e32 7, %4, implicit $exec
+ %14:vgpr_32 = V_ADD_U32_e32 8, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %15, implicit %16, implicit %17, implicit %18, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14
+...
+
+---
+name: waw_window_expires
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_window_expires
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr27 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr28 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 5, killed $vgpr24, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr27, implicit killed renamable $vgpr28, implicit killed renamable $vgpr16
+ ;
+ ; NOAH-LABEL: name: waw_window_expires
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, $vgpr24, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr20 = V_ADD_U32_e32 5, killed $vgpr24, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr20
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ %11:vgpr_32 = V_ADD_U32_e32 5, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11
+...
+
+---
+name: waw_second_wmma_flushes_family
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_second_wmma_flushes_family
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr48 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr40 = V_ADD_U32_e32 1, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 2, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 3, $vgpr48, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr43 = V_ADD_U32_e32 4, killed $vgpr48, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr40, implicit killed renamable $vgpr41, implicit killed renamable $vgpr42, implicit killed renamable $vgpr43
+ ;
+ ; NOAH-LABEL: name: waw_second_wmma_flushes_family
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr48 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 8, killed $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr32 = V_ADD_U32_e32 1, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr33 = V_ADD_U32_e32 2, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr34 = V_ADD_U32_e32 3, $vgpr48, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr35 = V_ADD_U32_e32 4, killed $vgpr48, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, implicit killed renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit killed renamable $vgpr32, implicit killed renamable $vgpr33, implicit killed renamable $vgpr34, implicit killed renamable $vgpr35
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %2:vreg_256_align2 = IMPLICIT_DEF
+ %3:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %15:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %16:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %2, %3, 8, %15, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_non_xdl_wmma_flushes_family
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: waw_non_xdl_wmma_flushes_family
+ ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr34_vgpr35 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr36 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr32_vgpr33, $vgpr34_vgpr35, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr24 = V_ADD_U32_e32 1, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr25 = V_ADD_U32_e32 2, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr26 = V_ADD_U32_e32 3, $vgpr36, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr16 = V_ADD_U32_e32 4, killed $vgpr36, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35, implicit killed renamable $vgpr24, implicit killed renamable $vgpr25, implicit killed renamable $vgpr26, implicit killed renamable $vgpr16
+ ;
+ ; NOAH-LABEL: name: waw_non_xdl_wmma_flushes_family
+ ; NOAH: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr32_vgpr33 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr34_vgpr35 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr36 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = IMPLICIT_DEF
+ ; NOAH-NEXT: renamable $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: dead early-clobber renamable $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23 = V_WMMA_F32_16X16X4_F32_w32_twoaddr $vgpr32_vgpr33, $vgpr34_vgpr35, 8, killed $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, 0, implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: V_NOP_e32 implicit $exec
+ ; NOAH-NEXT: renamable $vgpr16 = V_ADD_U32_e32 1, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr17 = V_ADD_U32_e32 2, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr18 = V_ADD_U32_e32 3, $vgpr36, implicit $exec
+ ; NOAH-NEXT: renamable $vgpr19 = V_ADD_U32_e32 4, killed $vgpr36, implicit $exec
+ ; NOAH-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr32_vgpr33, implicit killed renamable $vgpr34_vgpr35, implicit killed renamable $vgpr16, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+ %3:vreg_64_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %15:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ %16:vreg_256_align2 = V_WMMA_F32_16X16X4_F32_w32_twoaddr %2, %3, 8, %15, 0, 0, 0, 0, implicit $exec
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %7, implicit %8, implicit %9, implicit %10
+...
+
+---
+name: waw_cross_block_no_hints
+tracksRegLiveness: true
+body: |
+ ; SAME-LABEL: name: waw_cross_block_no_hints
+ ; SAME: bb.0:
+ ; SAME-NEXT: successors: %bb.1(0x80000000)
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; SAME-NEXT: renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = IMPLICIT_DEF
+ ; SAME-NEXT: dead early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 8, killed $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25, 0, 0, 0, 0, implicit $exec
+ ; SAME-NEXT: S_BRANCH %bb.1
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: bb.1:
+ ; SAME-NEXT: liveins: $vgpr16, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+ ; SAME-NEXT: {{ $}}
+ ; SAME-NEXT: renamable $vgpr17 = V_ADD_U32_e32 1, $vgpr16, implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: V_NOP_e32 implicit $exec
+ ; SAME-NEXT: renamable $vgpr18 = V_ADD_U32_e32 2, $vgpr16, implicit $exec
+ ; SAME-NEXT: renamable $vgpr19 = V_ADD_U32_e32 3, $vgpr16, implicit $exec
+ ; SAME-NEXT: renamable $vgpr16 = V_ADD_U32_e32 4, killed $vgpr16, implicit $exec
+ ; SAME-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit killed renamable $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit killed renamable $vgpr17, implicit killed renamable $vgpr18, implicit killed renamable $vgpr19, implicit killed renamable $vgpr16
+ bb.0:
+ successors: %bb.1
+
+ %0:vreg_256_align2 = IMPLICIT_DEF
+ %1:vreg_256_align2 = IMPLICIT_DEF
+ %4:vgpr_32 = IMPLICIT_DEF
+ %5:vreg_256_align2 = IMPLICIT_DEF
+ %6:vreg_256_align2 = V_WMMA_F32_16X16X32_BF16_w32_twoaddr %0, %1, 8, %5, 0, 0, 0, 0, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ %7:vgpr_32 = V_ADD_U32_e32 1, %4, implicit $exec
+ %8:vgpr_32 = V_ADD_U32_e32 2, %4, implicit $exec
+ %9:vgpr_32 = V_ADD_U32_e32 3, %4, implicit $exec
+ %10:vgpr_32 = V_ADD_U32_e32 4, %4, implicit $exec
+ S_ENDPGM 0, implicit %0, implicit %1, implicit %7, implicit %8, implicit %9, implicit %10
+...
diff --git a/llvm/test/CodeGen/AMDGPU/anyext-s16-to-s64.ll b/llvm/test/CodeGen/AMDGPU/anyext-s16-to-s64.ll
index 90c8780bf15d0b..ccc94ebe2538fa 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext-s16-to-s64.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext-s16-to-s64.ll
@@ -12,12 +12,11 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i16_index(<8 x i32> %a,
; SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; SDAG-FAKE16-NEXT: global_load_u16 v32, v[32:33], off
-; SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
-; SDAG-FAKE16-NEXT: v_mov_b32_e32 v33, 0
+; SDAG-FAKE16-NEXT: global_load_u16 v36, v[32:33], off
+; SDAG-FAKE16-NEXT: v_mov_b32_e32 v37, 0
; SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-FAKE16-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33]
+; SDAG-FAKE16-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[36:37]
; SDAG-FAKE16-NEXT: s_clause 0x1
; SDAG-FAKE16-NEXT: global_store_b128 v[34:35], v[28:31], off offset:16
; SDAG-FAKE16-NEXT: global_store_b128 v[34:35], v[24:27], off
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 883bf28c8cbd86..092220125e9c11 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -14,35 +14,35 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
+; SDAG-NEXT: global_load_b32 v12, v[8:9], off offset:44
+; SDAG-NEXT: global_load_b32 v13, v[0:1], off offset:4
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
-; SDAG-NEXT: global_load_b32 v10, v[8:9], off offset:44
-; SDAG-NEXT: global_load_b32 v11, v[0:1], off offset:4
+; SDAG-NEXT: v_add_nc_u64_e32 v[10:11], 0x58, v[8:9]
; SDAG-NEXT: ; wave barrier
+; SDAG-NEXT: s_mov_b32 s0, 0x58
+; SDAG-NEXT: s_wait_loadcnt 0x0
+; SDAG-NEXT: v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
-; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
-; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; SDAG-NEXT: global_load_b32 v15, v[0:1], off offset:8
; SDAG-NEXT: ; wave barrier
-; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
+; SDAG-NEXT: global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT: ; wave barrier
-; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
+; SDAG-NEXT: global_load_b32 v16, v[8:9], off offset:48
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
; SDAG-NEXT: s_wait_asynccnt 0x1
-; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
+; SDAG-NEXT: ds_load_b32 v17, v2 offset:84
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
-; SDAG-NEXT: s_wait_loadcnt 0x2
-; SDAG-NEXT: v_add_nc_u32_e32 v4, v11, v10
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; SDAG-NEXT: v_add3_u32 v0, v4, v3, v0
+; SDAG-NEXT: v_add3_u32 v12, v12, v17, v15
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; SDAG-NEXT: v_add3_u32 v0, v0, v1, v2
-; SDAG-NEXT: global_store_b32 v[6:7], v0, off
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-NEXT: v_add3_u32 v2, v12, v16, v2
+; SDAG-NEXT: global_store_b32 v[6:7], v2, off
; SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: interleaved_with_wave_barrier:
@@ -52,38 +52,38 @@ define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(
; GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
; GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x54, v0
+; GISEL-NEXT: global_load_b32 v12, v[8:9], off offset:44
+; GISEL-NEXT: global_load_b32 v13, v[0:1], off offset:4
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
-; GISEL-NEXT: global_load_b32 v10, v[8:9], off offset:44
-; GISEL-NEXT: global_load_b32 v11, v[0:1], off offset:4
+; GISEL-NEXT: v_add_co_u32 v10, vcc_lo, 0x58, v8
; GISEL-NEXT: ; wave barrier
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v9, vcc_lo
+; GISEL-NEXT: s_mov_b32 s0, 0x58
+; GISEL-NEXT: s_wait_loadcnt 0x0
+; GISEL-NEXT: v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
-; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
-; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
-; GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:8
+; GISEL-NEXT: global_load_b32 v15, v[0:1], off offset:8
; GISEL-NEXT: ; wave barrier
-; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
+; GISEL-NEXT: global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT: ; wave barrier
-; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
+; GISEL-NEXT: global_load_b32 v16, v[8:9], off offset:48
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_wait_asynccnt 0x1
-; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
+; GISEL-NEXT: ds_load_b32 v17, v2 offset:84
; GISEL-NEXT: ; wait_asyncmark(0)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
-; GISEL-NEXT: s_wait_loadcnt 0x2
-; GISEL-NEXT: v_add_nc_u32_e32 v6, v11, v10
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
-; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GISEL-NEXT: v_add3_u32 v0, v6, v3, v0
+; GISEL-NEXT: v_add3_u32 v12, v12, v17, v15
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GISEL-NEXT: v_add3_u32 v0, v0, v1, v2
-; GISEL-NEXT: global_store_b32 v[4:5], v0, off
+; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT: v_add3_u32 v2, v12, v16, v2
+; GISEL-NEXT: global_store_b32 v[4:5], v2, off
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
; First batch: global load, global load, async global-to-LDS
@@ -143,35 +143,33 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-NEXT: s_wait_kmcnt 0x0
-; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; SDAG-NEXT: s_add_co_i32 s6, s2, 4
-; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; SDAG-NEXT: v_dual_mov_b32 v2, 4 :: v_dual_mov_b32 v3, s6
+; SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 4
+; SDAG-NEXT: v_mov_b32_e32 v4, s6
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b32 s7, s2
-; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
+; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT: ; asyncmark
-; SDAG-NEXT: global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
-; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[0:1] offset:4 nv
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB1_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
; SDAG-NEXT: s_add_co_i32 s6, s6, 1
-; SDAG-NEXT: v_mov_b32_e32 v2, s8
+; SDAG-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, s7
+; SDAG-NEXT: s_add_co_i32 s7, s7, 4
+; SDAG-NEXT: s_cmp_lt_i32 s6, s3
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
-; SDAG-NEXT: v_mov_b32_e32 v2, s7
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(2)
; SDAG-NEXT: s_wait_asynccnt 0x2
-; SDAG-NEXT: s_add_co_i32 s7, s7, 4
-; SDAG-NEXT: s_cmp_lt_i32 s6, s3
-; SDAG-NEXT: ds_load_b32 v2, v2
+; SDAG-NEXT: ds_load_b32 v3, v3
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v3
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
@@ -217,15 +215,14 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
; GISEL-NEXT: s_add_co_i32 s7, s7, 1
-; GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GISEL-NEXT: v_dual_mov_b32 v1, s9 :: v_dual_mov_b32 v2, s8
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GISEL-NEXT: v_mov_b32_e32 v1, s8
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_wait_asynccnt 0x2
-; GISEL-NEXT: ds_load_b32 v1, v1
+; GISEL-NEXT: ds_load_b32 v2, v2
; GISEL-NEXT: s_wait_dscnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s9, v1
+; GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GISEL-NEXT: s_add_co_i32 s6, s6, s9
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
@@ -319,21 +316,21 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 4
+; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v7, 4
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: s_add_co_i32 s7, s10, 4
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; SDAG-NEXT: v_dual_mov_b32 v1, s10 :: v_dual_mov_b32 v4, s7
+; SDAG-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s7
; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0
; SDAG-NEXT: s_load_b32 s12, s[0:1], 0x0
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
; SDAG-NEXT: s_clause 0x2
-; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
+; SDAG-NEXT: global_load_async_to_lds_b32 v8, v0, s[8:9] offset:4 nv
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
-; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
+; SDAG-NEXT: global_load_async_to_lds_b32 v9, v7, s[8:9] offset:4 nv
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_mov_b32 s8, 2
@@ -349,17 +346,17 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v4, s12
-; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, s9
+; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
+; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v10, s9
; SDAG-NEXT: global_load_b32 v3, v0, s[4:5]
-; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
+; SDAG-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_add_nc_u32 v11, v5, v6
; SDAG-NEXT: s_clause 0x1
-; SDAG-NEXT: global_load_async_to_lds_b32 v4, v0, s[4:5] offset:4 nv
+; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: global_load_b32 v4, v0, s[0:1]
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(2)
; SDAG-NEXT: s_wait_asynccnt 0x2
-; SDAG-NEXT: ds_load_b32 v9, v9
+; SDAG-NEXT: ds_load_b32 v10, v10
; SDAG-NEXT: v_mov_b32_e32 v5, v1
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
@@ -369,8 +366,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u32_e32 v9, v10, v9
-; SDAG-NEXT: global_store_b32 v0, v9, s[6:7]
+; SDAG-NEXT: v_add_nc_u32_e32 v10, v11, v10
+; SDAG-NEXT: global_store_b32 v0, v10, s[6:7]
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
@@ -384,19 +381,15 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; SDAG-NEXT: ds_load_b32 v1, v0
; SDAG-NEXT: v_mov_b32_e32 v5, s0
; SDAG-NEXT: s_add_co_i32 s0, s11, -1
-; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: v_add_nc_u32_e32 v1, v2, v1
+; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; SDAG-NEXT: v_dual_add_nc_u32 v1, v2, v1 :: v_dual_add_nc_u32 v2, v3, v4
; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
-; SDAG-NEXT: s_wait_loadcnt 0x0
-; SDAG-NEXT: s_wait_xcnt 0x0
-; SDAG-NEXT: v_add_nc_u32_e32 v1, v3, v4
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_add_nc_u32 v0, v1, v0
-; SDAG-NEXT: global_store_b32 v1, v0, s[2:3] scale_offset
+; SDAG-NEXT: v_dual_add_nc_u32 v0, v2, v0 :: v_dual_mov_b32 v2, s0
+; SDAG-NEXT: global_store_b32 v2, v0, s[2:3] scale_offset
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_pipelined_loop_with_global:
@@ -417,14 +410,14 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: v_mov_b32_e32 v1, s10
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s6
+; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v5, s6
; GISEL-NEXT: s_add_co_u32 s6, s8, 8
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT: ; asyncmark
-; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
-; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
-; GISEL-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
+; GISEL-NEXT: global_load_b32 v2, v0, s[8:9] offset:4
+; GISEL-NEXT: global_load_b32 v3, v0, s[0:1] offset:4
+; GISEL-NEXT: global_load_async_to_lds_b32 v5, v4, s[8:9] offset:4 nv
; GISEL-NEXT: s_add_co_ci_u32 s7, s9, 0
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
@@ -433,41 +426,41 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: s_wait_loadcnt 0x1
-; GISEL-NEXT: v_readfirstlane_b32 s8, v1
+; GISEL-NEXT: v_readfirstlane_b32 s8, v2
; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s15, v2
+; GISEL-NEXT: v_readfirstlane_b32 s15, v3
; GISEL-NEXT: s_mov_b32 s9, s8
; GISEL-NEXT: s_mov_b32 s19, s15
; GISEL-NEXT: .LBB2_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s17, s14, 8
; GISEL-NEXT: global_load_b32 v2, v0, s[6:7]
-; GISEL-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v3, s14
+; GISEL-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v4, s14
; GISEL-NEXT: s_mov_b32 s18, s9
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: s_add_co_i32 s9, s12, s16
; GISEL-NEXT: s_add_co_i32 s13, s13, 1
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[6:7] offset:4 nv
-; GISEL-NEXT: global_load_b32 v1, v0, s[0:1]
+; GISEL-NEXT: global_load_b32 v3, v0, s[0:1]
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_wait_asynccnt 0x2
-; GISEL-NEXT: ds_load_b32 v3, v3
+; GISEL-NEXT: ds_load_b32 v4, v4
; GISEL-NEXT: s_mov_b32 s17, s19
; GISEL-NEXT: s_mov_b32 s12, s8
; GISEL-NEXT: s_wait_dscnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s16, v3
+; GISEL-NEXT: v_readfirstlane_b32 s16, v4
; GISEL-NEXT: s_add_co_i32 s9, s9, s16
; GISEL-NEXT: s_wait_xcnt 0x1
; GISEL-NEXT: s_add_co_u32 s6, s6, 4
-; GISEL-NEXT: v_mov_b32_e32 v3, s9
+; GISEL-NEXT: v_mov_b32_e32 v4, s9
; GISEL-NEXT: s_add_co_ci_u32 s7, s7, 0
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: s_mov_b32 s16, s15
-; GISEL-NEXT: global_store_b32 v0, v3, s[4:5]
+; GISEL-NEXT: global_store_b32 v0, v4, s[4:5]
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
@@ -476,7 +469,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s9, v2
; GISEL-NEXT: s_wait_loadcnt 0x0
-; GISEL-NEXT: v_readfirstlane_b32 s19, v1
+; GISEL-NEXT: v_readfirstlane_b32 s19, v3
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
@@ -503,9 +496,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: s_add_co_i32 s0, s1, s0
; GISEL-NEXT: s_add_co_i32 s1, s11, -1
-; GISEL-NEXT: s_wait_xcnt 0x0
-; GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GISEL-NEXT: global_store_b32 v1, v0, s[2:3] scale_offset
+; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
+; GISEL-NEXT: global_store_b32 v3, v0, s[2:3] scale_offset
; GISEL-NEXT: s_endpgm
prolog:
; Load first iteration
@@ -589,14 +582,14 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_asynccnt 0x0
-; SDAG-NEXT: ds_load_b32 v0, v2
-; SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; SDAG-NEXT: ds_load_b32 v3, v2
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: global_store_b32 v[4:5], v3, off
; SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: consecutive_asyncmarks:
@@ -604,14 +597,14 @@ define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:4
+; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(0)
; GISEL-NEXT: s_wait_asynccnt 0x0
-; GISEL-NEXT: ds_load_b32 v0, v2
-; GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GISEL-NEXT: ds_load_b32 v3, v2
; GISEL-NEXT: s_wait_dscnt 0x0
-; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: global_store_b32 v[6:7], v3, off
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
@@ -642,9 +635,9 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
; SDAG-NEXT: s_wait_asynccnt 0x1
-; SDAG-NEXT: ds_load_b32 v0, v2
+; SDAG-NEXT: ds_load_b32 v6, v2
; SDAG-NEXT: s_wait_dscnt 0x0
-; SDAG-NEXT: global_store_b32 v[4:5], v0, off
+; SDAG-NEXT: global_store_b32 v[4:5], v6, off
; SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: consecutive_asyncmarks_wait1:
@@ -661,9 +654,9 @@ define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_wait_asynccnt 0x1
-; GISEL-NEXT: ds_load_b32 v0, v2
+; GISEL-NEXT: ds_load_b32 v4, v2
; GISEL-NEXT: s_wait_dscnt 0x0
-; GISEL-NEXT: global_store_b32 v[6:7], v0, off
+; GISEL-NEXT: global_store_b32 v[6:7], v4, off
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index 67c115ac2ed767..af39dd48b744be 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -1742,28 +1742,26 @@ define amdgpu_kernel void @add_i32_varying_offset(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: add_i32_varying_offset:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
@@ -3027,28 +3025,26 @@ define amdgpu_kernel void @sub_i32_varying_offset(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: sub_i32_varying_offset:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index f09555fde5d32a..9d9fa908c0e309 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -1253,28 +1253,26 @@ define amdgpu_kernel void @add_i32_varying_offset(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: add_i32_varying_offset:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2538,28 +2536,26 @@ define amdgpu_kernel void @sub_i32_varying_offset(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: sub_i32_varying_offset:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 83d8aa12d54aad..f2c1198e21f546 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -1271,28 +1271,26 @@ define amdgpu_kernel void @add_i32_varying_vindex(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: add_i32_varying_vindex:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2729,28 +2727,26 @@ define amdgpu_kernel void @sub_i32_varying_vindex(ptr addrspace(1) %out, ptr add
; GFX13W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13W64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX13W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W64-NEXT: v_mov_b32_e32 v0, 0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W64-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W64-NEXT: s_endpgm
;
; GFX13W32-LABEL: sub_i32_varying_vindex:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_mov_b32 s6, 0x3ff
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_bitop2_b32 v0, s6, v0 bitop3:0x40
+; GFX13W32-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX13W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX13W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX13W32-NEXT: v_mov_b32_e32 v0, 0
; GFX13W32-NEXT: s_wait_loadcnt 0x0
; GFX13W32-NEXT: s_wait_kmcnt 0x0
-; GFX13W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX13W32-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX13W32-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 14dfc9d8314e90..292a62aa5d409a 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -1580,16 +1580,28 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX11-NEXT: global_store_d16_hi_b16 v[2:3], v0, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: test_load_store_f32_to_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: global_store_b16 v[2:3], v0, off
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: test_load_store_f32_to_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v0, s0
+; GFX1250TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
+; GFX1250TRUE16-NEXT: global_store_b16 v[2:3], v0, off
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: test_load_store_f32_to_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v0, s0
+; GFX1250FAKE16-NEXT: global_store_b16 v[2:3], v4, off
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%val = load float, ptr addrspace(1) %in
%val.bf16 = fptrunc float %val to bfloat
store bfloat %val.bf16, ptr addrspace(1) %out
@@ -1749,28 +1761,52 @@ define void @test_load_store_f64_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX11-NEXT: global_store_d16_hi_b16 v[2:3], v0, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: test_load_store_f64_to_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
-; GFX1250-NEXT: v_cmp_gt_f64_e64 s0, |v[0:1]|, |v[4:5]|
-; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s0
-; GFX1250-NEXT: v_dual_add_nc_u32 v0, v6, v0 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s0, 1, v7
-; GFX1250-NEXT: s_or_b32 vcc_lo, vcc_lo, s0
-; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: global_store_b16 v[2:3], v0, off
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: test_load_store_f64_to_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250TRUE16-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX1250TRUE16-NEXT: v_cmp_gt_f64_e64 s0, |v[0:1]|, |v[4:5]|
+; GFX1250TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v4, -1, 1, s0
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v4, v6, v4 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 1, v7
+; GFX1250TRUE16-NEXT: s_or_b32 vcc_lo, vcc_lo, s0
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
+; GFX1250TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
+; GFX1250TRUE16-NEXT: global_store_b16 v[2:3], v0, off
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: test_load_store_f64_to_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX1250FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250FAKE16-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX1250FAKE16-NEXT: v_cmp_gt_f64_e64 s0, |v[0:1]|, |v[4:5]|
+; GFX1250FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v4, -1, 1, s0
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v4, v6, v4 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e64 s0, 1, v7
+; GFX1250FAKE16-NEXT: s_or_b32 vcc_lo, vcc_lo, s0
+; GFX1250FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
+; GFX1250FAKE16-NEXT: global_store_b16 v[2:3], v4, off
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%val = load double, ptr addrspace(1) %in
%val.bf16 = fptrunc double %val to bfloat
store bfloat %val.bf16, ptr addrspace(1) %out
@@ -1850,9 +1886,8 @@ define void @test_load_store_bf16_to_f32(ptr addrspace(1) %in, ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u16 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-NEXT: global_store_b32 v[2:3], v0, off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX1250-NEXT: global_store_b32 v[2:3], v4, off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%val = load bfloat, ptr addrspace(1) %in
%val.f32 = fpext bfloat %val to float
@@ -1940,11 +1975,10 @@ define void @test_load_store_bf16_to_f64(ptr addrspace(1) %in, ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u16 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
+; GFX1250-NEXT: global_store_b64 v[2:3], v[4:5], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%val = load bfloat, ptr addrspace(1) %in
%val.f64 = fpext bfloat %val to double
@@ -6725,11 +6759,11 @@ define <2 x double> @global_extload_v2bf16_to_v2f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v0 :: v_dual_bitop2_b32 v3, s0, v0 bitop3:0x40
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v0 :: v_dual_bitop2_b32 v2, s0, v0 bitop3:0x40
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%load = load <2 x bfloat>, ptr addrspace(1) %ptr
%fpext = fpext <2 x bfloat> %load to <2 x double>
@@ -6967,16 +7001,15 @@ define <4 x double> @global_extload_v4bf16_to_v4f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: global_load_b64 v[2:3], v[0:1], off
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_bitop2_b32 v2, s0, v2 bitop3:0x40
+; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v2 :: v_dual_bitop2_b32 v2, s0, v2 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v3 :: v_dual_bitop2_b32 v6, s0, v3 bitop3:0x40
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_dual_lshlrev_b32 v5, 16, v3 :: v_dual_bitop2_b32 v6, s0, v3 bitop3:0x40
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v4
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v5
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%load = load <4 x bfloat>, ptr addrspace(1) %ptr
@@ -7106,17 +7139,16 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_bitop2_b32 v2, s0, v2 bitop3:0x40
+; GFX1250-NEXT: v_dual_lshlrev_b32 v5, 16, v2 :: v_dual_bitop2_b32 v2, s0, v2 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_dual_lshlrev_b32 v5, 16, v3 :: v_dual_bitop2_b32 v6, s0, v3 bitop3:0x40
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_lshlrev_b32 v6, 16, v3 :: v_dual_bitop2_b32 v7, s0, v3 bitop3:0x40
; GFX1250-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v5
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v5
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v7
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%load = load <5 x bfloat>, ptr addrspace(1) %ptr
@@ -7254,15 +7286,15 @@ define <6 x double> @global_extload_v6bf16_to_v6f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: global_load_b96 v[4:6], v[0:1], off
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v4 :: v_dual_bitop2_b32 v2, s0, v4 bitop3:0x40
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v4 :: v_dual_bitop2_b32 v3, s0, v4 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v5 :: v_dual_bitop2_b32 v7, s0, v5 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: v_dual_lshlrev_b32 v8, 16, v6 :: v_dual_bitop2_b32 v10, s0, v6 bitop3:0x40
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v7
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
@@ -7427,8 +7459,7 @@ define <8 x double> @global_extload_v8bf16_to_v8f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: global_load_b128 v[8:11], v[0:1], off
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v8 :: v_dual_bitop2_b32 v2, s0, v8 bitop3:0x40
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v8 :: v_dual_bitop2_b32 v3, s0, v8 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v9 :: v_dual_bitop2_b32 v6, s0, v9 bitop3:0x40
@@ -7437,8 +7468,9 @@ define <8 x double> @global_extload_v8bf16_to_v8f64(ptr addrspace(1) %ptr) #0 {
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: v_dual_lshlrev_b32 v12, 16, v11 :: v_dual_bitop2_b32 v14, s0, v11 bitop3:0x40
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
@@ -7719,8 +7751,7 @@ define <16 x double> @global_extload_v16bf16_to_v16f64(ptr addrspace(1) %ptr) #0
; GFX1250-NEXT: global_load_b128 v[24:27], v[0:1], off offset:16
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v8 :: v_dual_bitop2_b32 v2, s0, v8 bitop3:0x40
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v8 :: v_dual_bitop2_b32 v3, s0, v8 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v9 :: v_dual_bitop2_b32 v6, s0, v9 bitop3:0x40
@@ -7739,8 +7770,9 @@ define <16 x double> @global_extload_v16bf16_to_v16f64(ptr addrspace(1) %ptr) #0
; GFX1250-NEXT: v_dual_lshlrev_b32 v24, 16, v26 :: v_dual_bitop2_b32 v26, s0, v26 bitop3:0x40
; GFX1250-NEXT: s_mov_b32 s0, 0xffff0000
; GFX1250-NEXT: v_dual_lshlrev_b32 v28, 16, v27 :: v_dual_bitop2_b32 v30, s0, v27 bitop3:0x40
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
@@ -9100,87 +9132,90 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX1250-NEXT: global_load_u16 v33, v[2:3], off offset:48
; GFX1250-NEXT: global_load_u16 v34, v[2:3], off offset:54
; GFX1250-NEXT: s_wait_loadcnt 0x1e
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v37, 16, v10
-; GFX1250-NEXT: s_wait_loadcnt 0x1b
-; GFX1250-NEXT: v_dual_lshlrev_b32 v85, 16, v4 :: v_dual_lshlrev_b32 v84, 16, v5
-; GFX1250-NEXT: s_wait_loadcnt 0x19
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v10, 16, v10
+; GFX1250-NEXT: s_wait_loadcnt 0x1c
+; GFX1250-NEXT: v_dual_lshlrev_b32 v81, 16, v6 :: v_dual_lshlrev_b32 v39, 16, v4
+; GFX1250-NEXT: s_wait_loadcnt 0x1a
+; GFX1250-NEXT: v_dual_lshlrev_b32 v38, 16, v5 :: v_dual_lshlrev_b32 v35, 16, v7
+; GFX1250-NEXT: s_wait_loadcnt 0x18
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v35, 16, v7 :: v_dual_lshlrev_b32 v2, 16, v8
-; GFX1250-NEXT: s_wait_loadcnt 0x17
-; GFX1250-NEXT: v_dual_lshlrev_b32 v3, 16, v9 :: v_dual_lshlrev_b32 v7, 16, v11
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v8 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX1250-NEXT: s_wait_loadcnt 0x16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v7, 16, v11 :: v_dual_lshlrev_b32 v11, 16, v12
; GFX1250-NEXT: s_wait_loadcnt 0x15
-; GFX1250-NEXT: v_dual_lshlrev_b32 v11, 16, v12 :: v_dual_lshlrev_b32 v12, 16, v13
-; GFX1250-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX1250-NEXT: v_lshlrev_b32_e32 v80, 16, v13
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v2
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v7
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v11
-; GFX1250-NEXT: s_wait_loadcnt 0x13
-; GFX1250-NEXT: v_dual_lshlrev_b32 v36, 16, v14 :: v_dual_lshlrev_b32 v38, 16, v15
+; GFX1250-NEXT: s_wait_loadcnt 0x11
+; GFX1250-NEXT: v_dual_lshlrev_b32 v69, 16, v17 :: v_dual_lshlrev_b32 v37, 16, v16
+; GFX1250-NEXT: s_wait_loadcnt 0xe
+; GFX1250-NEXT: v_dual_lshlrev_b32 v36, 16, v20 :: v_dual_lshlrev_b32 v82, 16, v18
; GFX1250-NEXT: s_wait_loadcnt 0xc
-; GFX1250-NEXT: v_dual_lshlrev_b32 v21, 16, v21 :: v_dual_lshlrev_b32 v39, 16, v22
-; GFX1250-NEXT: s_wait_loadcnt 0xb
-; GFX1250-NEXT: v_dual_lshlrev_b32 v49, 16, v23 :: v_dual_lshlrev_b32 v68, 16, v17
+; GFX1250-NEXT: v_dual_lshlrev_b32 v68, 16, v21 :: v_dual_lshlrev_b32 v22, 16, v22
; GFX1250-NEXT: s_wait_loadcnt 0x9
-; GFX1250-NEXT: v_dual_lshlrev_b32 v48, 16, v16 :: v_dual_lshlrev_b32 v25, 16, v25
-; GFX1250-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX1250-NEXT: s_wait_loadcnt 0x5
-; GFX1250-NEXT: v_dual_lshlrev_b32 v50, 16, v28 :: v_dual_lshlrev_b32 v64, 16, v29
+; GFX1250-NEXT: v_dual_lshlrev_b32 v53, 16, v24 :: v_dual_lshlrev_b32 v64, 16, v25
+; GFX1250-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX1250-NEXT: s_wait_loadcnt 0x7
+; GFX1250-NEXT: v_dual_lshlrev_b32 v27, 16, v27 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX1250-NEXT: s_wait_loadcnt 0x4
+; GFX1250-NEXT: v_dual_lshlrev_b32 v49, 16, v29 :: v_dual_lshlrev_b32 v52, 16, v30
; GFX1250-NEXT: s_wait_loadcnt 0x3
-; GFX1250-NEXT: v_dual_lshlrev_b32 v51, 16, v30 :: v_dual_lshlrev_b32 v52, 16, v31
-; GFX1250-NEXT: v_dual_lshlrev_b32 v69, 16, v27 :: v_dual_lshlrev_b32 v70, 16, v26
+; GFX1250-NEXT: v_dual_lshlrev_b32 v54, 16, v31 :: v_dual_lshlrev_b32 v48, 16, v28
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v53, 16, v34 :: v_dual_lshlrev_b32 v32, 16, v32
-; GFX1250-NEXT: v_lshlrev_b32_e32 v33, 16, v33
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[22:23], v38
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[30:31], v39
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[38:39], v50
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[54:55], v53
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[14:15], v35
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[28:29], v48
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[34:35], v49
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[48:49], v33
-; GFX1250-NEXT: v_dual_lshlrev_b32 v20, 16, v20 :: v_dual_lshlrev_b32 v81, 16, v18
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[66:67], v64
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[64:65], v25
-; GFX1250-NEXT: v_lshlrev_b32_e32 v80, 16, v19
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[18:19], v36
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[16:17], v37
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[36:37], v70
+; GFX1250-NEXT: v_dual_lshlrev_b32 v66, 16, v32 :: v_dual_lshlrev_b32 v34, 16, v34
+; GFX1250-NEXT: v_lshlrev_b32_e32 v65, 16, v33
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[54:55], v54
+; GFX1250-NEXT: v_dual_lshlrev_b32 v11, 16, v14 :: v_dual_lshlrev_b32 v14, 16, v15
+; GFX1250-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[50:51], v48
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[16:17], v35
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[28:29], v36
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[30:31], v37
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[36:37], v23
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[32:33], v22
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[70:71], v68
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[68:69], v69
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[24:25], v14
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[22:23], v15
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[20:21], v11
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[18:19], v10
; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off offset:240
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v51
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[50:51], v52
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[52:53], v32
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v24
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[32:33], v69
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[70:71], v21
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[68:69], v68
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[26:27], v20
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[24:25], v81
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[20:21], v80
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[82:83], v12
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[80:81], v13
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[12:13], v85
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[10:11], v1
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v52
; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:224
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v84
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v34
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v66
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v53
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[52:53], v65
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[66:67], v49
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[64:65], v64
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[48:49], v26
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[34:35], v27
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[26:27], v82
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[82:83], v80
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[80:81], v81
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[14:15], v39
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[12:13], v1
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[10:11], v38
; GFX1250-NEXT: s_clause 0xd
-; GFX1250-NEXT: scratch_store_b128 v0, v[52:55], off offset:208
-; GFX1250-NEXT: scratch_store_b128 v0, v[48:51], off offset:192
+; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:208
+; GFX1250-NEXT: scratch_store_b128 v0, v[52:55], off offset:192
; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off offset:176
; GFX1250-NEXT: scratch_store_b128 v0, v[64:67], off offset:160
-; GFX1250-NEXT: scratch_store_b128 v0, v[36:39], off offset:144
-; GFX1250-NEXT: scratch_store_b128 v0, v[32:35], off offset:128
-; GFX1250-NEXT: scratch_store_b128 v0, v[28:31], off offset:112
+; GFX1250-NEXT: scratch_store_b128 v0, v[48:51], off offset:144
+; GFX1250-NEXT: scratch_store_b128 v0, v[34:37], off offset:128
+; GFX1250-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
; GFX1250-NEXT: scratch_store_b128 v0, v[68:71], off offset:96
-; GFX1250-NEXT: scratch_store_b128 v0, v[24:27], off offset:80
-; GFX1250-NEXT: scratch_store_b128 v0, v[20:23], off offset:64
-; GFX1250-NEXT: scratch_store_b128 v0, v[16:19], off offset:48
+; GFX1250-NEXT: scratch_store_b128 v0, v[26:29], off offset:80
+; GFX1250-NEXT: scratch_store_b128 v0, v[22:25], off offset:64
+; GFX1250-NEXT: scratch_store_b128 v0, v[18:21], off offset:48
; GFX1250-NEXT: scratch_store_b128 v0, v[80:83], off offset:32
-; GFX1250-NEXT: scratch_store_b128 v0, v[12:15], off offset:16
-; GFX1250-NEXT: scratch_store_b128 v0, v[8:11], off
+; GFX1250-NEXT: scratch_store_b128 v0, v[14:17], off offset:16
+; GFX1250-NEXT: scratch_store_b128 v0, v[10:13], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%load = load <32 x bfloat>, ptr addrspace(1) %ptr
%fpext = fpext <32 x bfloat> %load to <32 x double>
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
index 79e9f643074f25..263fb62616b5d7 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
@@ -329,13 +329,12 @@ define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
; GCN-NEXT: v_nop
; GCN-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GCN-NEXT: s_mov_b32 s2, 0x3ff
-; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
+; GCN-NEXT: v_and_b32_e32 v3, 0x3ff, v0
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v3
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: global_load_b32 v2, v0, s[0:1] scale_offset scope:SCOPE_SYS
+; GCN-NEXT: global_load_b32 v2, v3, s[0:1] scale_offset scope:SCOPE_SYS
; GCN-NEXT: s_wait_loadcnt 0x0
-; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
; GCN-NEXT: s_mov_b32 s0, exec_lo
; GCN-NEXT: v_cmpx_ne_u32_e32 0, v2
@@ -367,13 +366,12 @@ define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-ADD-PC64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GCN-ADD-PC64-NEXT: s_mov_b32 s2, 0x3ff
-; GCN-ADD-PC64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
+; GCN-ADD-PC64-NEXT: v_and_b32_e32 v3, 0x3ff, v0
+; GCN-ADD-PC64-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v3
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: global_load_b32 v2, v0, s[0:1] scale_offset scope:SCOPE_SYS
+; GCN-ADD-PC64-NEXT: global_load_b32 v2, v3, s[0:1] scale_offset scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_loadcnt 0x0
-; GCN-ADD-PC64-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-ADD-PC64-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
; GCN-ADD-PC64-NEXT: s_mov_b32 s0, exec_lo
; GCN-ADD-PC64-NEXT: v_cmpx_ne_u32_e32 0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index deb186d6f68339..1b934f6f091fdf 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -3912,9 +3912,9 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1250-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX1250-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_u32_f32_e32 v0, v1
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
; GFX1250-NEXT: s_mul_i32 s4, s4, s3
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 97a0d6880c6b6c..aea29de71dfa52 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -757,35 +757,31 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: .LBB2_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_wait_dscnt 0x0
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; GCN-NEXT: v_nop
+; GCN-NEXT: s_wait_dscnt 0x4
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: v_mov_b32_e32 v72, s0
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15]
+; GCN-NEXT: s_add_co_i32 s0, s0, s1
+; GCN-NEXT: s_and_b32 s3, s2, exec_lo
; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13]
; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11]
-; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v60, s0
-; GCN-NEXT: s_add_co_i32 s0, s0, s1
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; GCN-NEXT: ds_load_tr16_b128 v[32:35], v60
-; GCN-NEXT: ds_load_tr16_b128 v[36:39], v60 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v60 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v60 offset:448
-; GCN-NEXT: s_and_b32 s3, s2, exec_lo
; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9]
; GCN-NEXT: s_cmp_lg_u32 s3, 1
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: ds_load_tr16_b128 v[32:35], v72
+; GCN-NEXT: ds_load_tr16_b128 v[36:39], v72 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v72 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v72 offset:192
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
+; GCN-NEXT: ds_load_tr16_b128 v[48:51], v72 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[52:55], v72 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[56:59], v72 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[60:63], v72 offset:448
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[64:71], v[64:71], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
@@ -1075,37 +1071,36 @@ define amdgpu_kernel void @ds_wmma_loop_carried(ptr addrspace(3) %base, ptr addr
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[56:63], v[64:71], v[24:31]
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
-; GCN-NEXT: .LBB3_1: ; %loop
-; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_wait_dscnt 0x0
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[56:63], v[24:31]
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15]
-; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13]
-; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11]
-; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9]
+; GCN-NEXT: .LBB3_1: ; %loop
+; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT: s_wait_dscnt 0x4
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[0:7], v[48:55], v[32:39]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[56:63], v[24:31]
-; GCN-NEXT: v_nop
+; GCN-NEXT: v_mov_b32_e32 v72, s0
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v60, s0
+; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15]
; GCN-NEXT: s_add_co_i32 s0, s0, s1
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[0:7], v[48:55], v[32:39]
-; GCN-NEXT: ds_load_tr16_b128 v[0:3], v60
-; GCN-NEXT: ds_load_tr16_b128 v[4:7], v60 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v60 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v60 offset:448
; GCN-NEXT: s_and_b32 s3, s2, exec_lo
+; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13]
+; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11]
+; GCN-NEXT: s_wait_dscnt 0x0
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[56:63], v[24:31]
; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9]
; GCN-NEXT: s_cmp_lg_u32 s3, 1
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[32:39], v[0:7], v[48:55], v[32:39]
+; GCN-NEXT: ds_load_tr16_b128 v[0:3], v72
+; GCN-NEXT: ds_load_tr16_b128 v[4:7], v72 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[48:51], v72 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[52:55], v72 offset:192
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[56:63], v[24:31]
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v72 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v72 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[56:59], v72 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[60:63], v72 offset:448
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[64:71], v[64:71], v[16:23]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[64:71], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[64:71], v[64:71], v[16:23]
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
index 65a2a008aae68b..36b279d3f541f3 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: llc -mtriple=amdgpu12.50--amdpal -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GFX1250,GFX1250-UNALIGNED %s
; RUN: llc -mtriple=amdgpu12.50s--amdpal -mattr=+load-store-opt,+unaligned-access-mode < %s | FileCheck -enable-var-scope -check-prefixes=GFX1250,GFX1250S-UNALIGNED %s
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index d76e2445cbf48a..bdd465499a3bc2 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -352,12 +352,12 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(ptr addrspace(1) %C
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 2, v2
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v2, s[0:1] scale_offset
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 2, v2
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ds_store_2addr_b32 v2, v0, v1 offset1:8
+; GFX1250-NEXT: ds_store_2addr_b32 v3, v0, v1 offset1:8
; GFX1250-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr <2 x float>, ptr addrspace(1) %in, i32 %x.i
@@ -755,12 +755,11 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(ptr addrspace(1) %C
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1] scale_offset
; GFX1250-NEXT: global_load_b32 v2, v0, s[2:3] scale_offset
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s5
+; GFX1250-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s5
; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: ds_store_b32 v0, v1 offset:32
+; GFX1250-NEXT: ds_store_b32 v3, v1 offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ds_store_b32 v3, v2 offset:32
+; GFX1250-NEXT: ds_store_b32 v4, v2 offset:32
; GFX1250-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in0.gep = getelementptr float, ptr addrspace(1) %in0, i32 %x.i
@@ -877,11 +876,10 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
; GFX1250-UNALIGNED-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
; GFX1250-UNALIGNED-NEXT: s_wait_kmcnt 0x0
; GFX1250-UNALIGNED-NEXT: global_load_b64 v[0:1], v2, s[0:1]
-; GFX1250-UNALIGNED-NEXT: s_wait_xcnt 0x0
-; GFX1250-UNALIGNED-NEXT: v_add_nc_u32_e32 v2, s2, v2
+; GFX1250-UNALIGNED-NEXT: v_add_nc_u32_e32 v3, s2, v2
; GFX1250-UNALIGNED-NEXT: s_wait_loadcnt 0x0
-; GFX1250-UNALIGNED-NEXT: ds_store_2addr_b32 v2, v0, v1 offset1:1
-; GFX1250-UNALIGNED-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:14 offset1:15
+; GFX1250-UNALIGNED-NEXT: ds_store_2addr_b32 v3, v0, v1 offset1:1
+; GFX1250-UNALIGNED-NEXT: ds_store_2addr_b32 v3, v0, v1 offset0:14 offset1:15
; GFX1250-UNALIGNED-NEXT: s_endpgm
;
; GFX1250S-UNALIGNED-LABEL: misaligned_simple_write2_one_val_f64:
@@ -896,11 +894,10 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
; GFX1250S-UNALIGNED-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
; GFX1250S-UNALIGNED-NEXT: s_wait_kmcnt 0x0
; GFX1250S-UNALIGNED-NEXT: global_load_b64 v[0:1], v2, s[0:1]
-; GFX1250S-UNALIGNED-NEXT: s_wait_xcnt 0x0
-; GFX1250S-UNALIGNED-NEXT: v_add_nc_u32_e32 v2, s2, v2
+; GFX1250S-UNALIGNED-NEXT: v_add_nc_u32_e32 v3, s2, v2
; GFX1250S-UNALIGNED-NEXT: s_wait_loadcnt 0x0
-; GFX1250S-UNALIGNED-NEXT: ds_store_b64 v2, v[0:1]
-; GFX1250S-UNALIGNED-NEXT: ds_store_b64 v2, v[0:1] offset:56
+; GFX1250S-UNALIGNED-NEXT: ds_store_b64 v3, v[0:1]
+; GFX1250S-UNALIGNED-NEXT: ds_store_b64 v3, v[0:1] offset:56
; GFX1250S-UNALIGNED-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr double, ptr addrspace(1) %in, i32 %x.i
@@ -999,11 +996,10 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
; GFX1250-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v2, s[0:1]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v2, s2, v2
+; GFX1250-NEXT: v_add_nc_u32_e32 v3, s2, v2
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ds_store_b64 v2, v[0:1] offset:5
-; GFX1250-NEXT: ds_store_b64 v2, v[0:1] offset:9
+; GFX1250-NEXT: ds_store_b64 v3, v[0:1] offset:5
+; GFX1250-NEXT: ds_store_b64 v3, v[0:1] offset:9
; GFX1250-NEXT: s_endpgm
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr double, ptr addrspace(1) %in, i32 %x.i
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index f559943070dd77..db3d261919b13f 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -727,8 +727,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2bf16(ptr addrspace(1) %out)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_pk_mul_bf16 v2, 1.0, v0 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX1250-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -748,13 +748,13 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2bf16(ptr addrspace(1)
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
-; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -775,13 +775,13 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2bf16(ptr addrspac
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
-; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -807,8 +807,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2bf16(ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_pk_mul_bf16 v2, 1.0, v0 op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX1250-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index ff988c73a0dbc5..5a5a57bc148e2a 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -7569,12 +7569,12 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-NEXT: s_mov_b32 s2, 0x3ff
; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1251-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_bitop2_b32 v0, s2, v0 bitop3:0x40
+; GFX1251-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_bitop2_b32 v4, s2, v0 bitop3:0x40
; GFX1251-NEXT: s_wait_kmcnt 0x0
-; GFX1251-NEXT: global_load_b128 v[0:3], v0, s[0:1] scale_offset
+; GFX1251-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1251-NEXT: s_wait_loadcnt 0x0
; GFX1251-NEXT: v_pk_max_num_f64 v[0:3], v[0:3], v[0:3]
-; GFX1251-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX1251-NEXT: global_store_b128 v5, v[0:3], s[0:1]
; GFX1251-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x double>, ptr addrspace(1) %out, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll b/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
index 4fed18d6eb0d87..2f7bdae9c418ec 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
@@ -18,12 +18,12 @@ define amdgpu_kernel void @test_move_load_address_to_vgpr(ptr addrspace(1) nocap
; GCN-NEXT: v_nop
; GCN-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: v_mov_b32_e32 v5, 0
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS
+; GCN-NEXT: global_load_b32 v4, v5, s[0:1] scope:SCOPE_SYS
; GCN-NEXT: s_wait_loadcnt 0x0
-; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[2:3]
-; GCN-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v2
+; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[4:5]
+; GCN-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v4
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GCN-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GCN-NEXT: .LBB0_1: ; %bb3
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 5fc001c8485750..bd47c2ffe12554 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -1471,13 +1471,12 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase,
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[2:3]
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 2, s[2:3]
+; GFX1250-NEXT: flat_load_b32 v0, v[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
%voffset = load i32, ptr %voffset.ptr
@@ -1495,13 +1494,12 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inr
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[2:3] offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
@@ -1511,14 +1509,13 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inr
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[2:3] offset:128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
@@ -1528,13 +1525,12 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inr
; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NOECC-NEXT: s_wait_xcnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[2:3] offset:128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%voffset = load i32, ptr %voffset.ptr
@@ -1555,10 +1551,10 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, pt
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[2:3]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
@@ -1568,15 +1564,14 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, pt
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
@@ -1588,10 +1583,10 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, pt
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
@@ -1611,10 +1606,10 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[2:3] offset:400
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
@@ -1624,15 +1619,14 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[2:3] offset:400
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
@@ -1644,10 +1638,10 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[2:3]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[2:3] offset:400
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
@@ -1696,19 +1690,33 @@ define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
@@ -1756,19 +1764,33 @@ define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_immneg128:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_immneg128:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
@@ -4549,9 +4571,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4614,9 +4636,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4667,8 +4689,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -4681,9 +4703,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -4696,8 +4718,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -4710,9 +4732,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -4747,8 +4769,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -4761,9 +4783,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -4776,8 +4798,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -4790,9 +4812,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -4828,8 +4850,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -4857,8 +4879,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[4:5]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -4893,8 +4915,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -4922,8 +4944,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[4:5] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -4959,8 +4981,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -4988,8 +5010,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[4:5]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5025,8 +5047,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5054,8 +5076,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[4:5] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5092,8 +5114,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5123,8 +5145,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[4:5]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5177,8 +5199,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5208,8 +5230,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[4:5] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5267,8 +5289,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5280,14 +5302,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
@@ -5299,8 +5319,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5313,9 +5333,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -5349,8 +5369,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5362,14 +5382,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
@@ -5381,8 +5399,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5395,9 +5413,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -5432,8 +5450,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5446,9 +5464,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -5461,10 +5479,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5513,8 +5530,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5527,9 +5544,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -5542,10 +5559,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1] offset:-128
-; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -5595,8 +5611,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5663,8 +5679,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5732,8 +5748,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5801,8 +5817,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5871,8 +5887,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[4:5]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -5959,8 +5975,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[4:5] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 477c58bef75427..0e90b55dabd326 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -13,13 +13,12 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voff
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v4, v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[4:5]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[4:5], v2
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr:
@@ -28,14 +27,13 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voff
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[4:5], v2
; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -52,13 +50,12 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbas
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v4, v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:2047
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[4:5]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[4:5], v2 offset:2047
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:
@@ -67,14 +64,13 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbas
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2 offset:2047
+; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[4:5], v2 offset:2047
; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -92,13 +88,12 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %s
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: flat_load_b32 v4, v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:-2048
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[4:5]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[4:5], v2 offset:-2048
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -107,14 +102,13 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %s
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2 offset:-2048
+; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[4:5], v2 offset:-2048
; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
index 768b1d5cfdbda0..ecc5ba767c09e2 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
@@ -317,10 +317,10 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f32_off4_slc(ptr addrsp
; GFX1310-NEXT: s_mov_b32 s4, 4
; GFX1310-NEXT: s_wait_kmcnt 0x0
; GFX1310-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1310-NEXT: v_mov_b32_e32 v2, s10
; GFX1310-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; GFX1310-NEXT: v_mov_b32_e32 v1, s10
; GFX1310-NEXT: s_wait_loadcnt 0x0
-; GFX1310-NEXT: ds_store_b32 v1, v0
+; GFX1310-NEXT: ds_store_b32 v2, v0
; GFX1310-NEXT: s_endpgm
;
; G_SI-LABEL: raw_ptr_buffer_atomic_min_rtn_f32_off4_slc:
@@ -411,10 +411,10 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f32_off4_slc(ptr addrsp
; G_GFX1310-NEXT: s_mov_b32 s4, 4
; G_GFX1310-NEXT: s_wait_kmcnt 0x0
; G_GFX1310-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; G_GFX1310-NEXT: v_mov_b32_e32 v2, s10
; G_GFX1310-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; G_GFX1310-NEXT: v_mov_b32_e32 v1, s10
; G_GFX1310-NEXT: s_wait_loadcnt 0x0
-; G_GFX1310-NEXT: ds_store_b32 v1, v0
+; G_GFX1310-NEXT: ds_store_b32 v2, v0
; G_GFX1310-NEXT: s_endpgm
; GFX1010-LABEL: raw_ptr_buffer_atomic_min_rtn_f32_off4_slc:
main_body:
@@ -720,12 +720,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f32_off4_slc(ptr addrsp
; GFX1310: ; %bb.0: ; %main_body
; GFX1310-NEXT: s_load_b256 s[0:7], s[4:5], 0x24 nv
; GFX1310-NEXT: s_wait_kmcnt 0x0
-; GFX1310-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1310-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s4
+; GFX1310-NEXT: v_mov_b32_e32 v1, s5
; GFX1310-NEXT: s_mov_b32 s4, 4
; GFX1310-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; GFX1310-NEXT: v_mov_b32_e32 v1, 0
; GFX1310-NEXT: s_wait_loadcnt 0x0
-; GFX1310-NEXT: global_store_b32 v1, v0, s[6:7]
+; GFX1310-NEXT: global_store_b32 v2, v0, s[6:7]
; GFX1310-NEXT: s_endpgm
;
; G_SI-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
@@ -795,12 +795,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f32_off4_slc(ptr addrsp
; G_GFX1310: ; %bb.0: ; %main_body
; G_GFX1310-NEXT: s_load_b256 s[0:7], s[4:5], 0x24 nv
; G_GFX1310-NEXT: s_wait_kmcnt 0x0
-; G_GFX1310-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; G_GFX1310-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s4
+; G_GFX1310-NEXT: v_mov_b32_e32 v1, s5
; G_GFX1310-NEXT: s_mov_b32 s4, 4
; G_GFX1310-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; G_GFX1310-NEXT: v_mov_b32_e32 v1, 0
; G_GFX1310-NEXT: s_wait_loadcnt 0x0
-; G_GFX1310-NEXT: global_store_b32 v1, v0, s[6:7]
+; G_GFX1310-NEXT: global_store_b32 v2, v0, s[6:7]
; G_GFX1310-NEXT: s_endpgm
main_body:
%ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index bac606ecf20773..3e887f1bc251aa 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -132,15 +132,14 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -263,15 +262,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -393,15 +391,14 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -523,15 +520,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -654,15 +650,14 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -785,15 +780,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -915,15 +909,14 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1045,15 +1038,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1176,15 +1168,14 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
@@ -1307,15 +1298,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
@@ -1437,15 +1427,14 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
@@ -1567,15 +1556,14 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s10
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
index e9487127b23f3e..fb8e9971c6286c 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
@@ -571,12 +571,11 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
; GFX1310-NEXT: s_load_b256 s[0:7], s[4:5], 0x24 nv
; GFX1310-NEXT: s_wait_kmcnt 0x0
; GFX1310-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX1310-NEXT: v_mov_b32_e32 v2, s6
+; GFX1310-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX1310-NEXT: s_mov_b32 s4, 4
; GFX1310-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; GFX1310-NEXT: v_mov_b32_e32 v2, s7
; GFX1310-NEXT: s_wait_loadcnt 0x0
-; GFX1310-NEXT: ds_store_b64 v2, v[0:1]
+; GFX1310-NEXT: ds_store_b64 v3, v[0:1]
; GFX1310-NEXT: s_endpgm
;
; G_SI-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
@@ -638,12 +637,11 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
; G_GFX1310-NEXT: s_load_b256 s[0:7], s[4:5], 0x24 nv
; G_GFX1310-NEXT: s_wait_kmcnt 0x0
; G_GFX1310-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; G_GFX1310-NEXT: v_mov_b32_e32 v2, s6
+; G_GFX1310-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; G_GFX1310-NEXT: s_mov_b32 s4, 4
; G_GFX1310-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN
-; G_GFX1310-NEXT: v_mov_b32_e32 v2, s7
; G_GFX1310-NEXT: s_wait_loadcnt 0x0
-; G_GFX1310-NEXT: ds_store_b64 v2, v[0:1]
+; G_GFX1310-NEXT: ds_store_b64 v3, v[0:1]
; G_GFX1310-NEXT: s_endpgm
main_body:
%ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index 8ac3bdd90231e2..20e8e1e7371bda 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -17,14 +17,13 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-FAKE16-NEXT: global_load_u8 v7, v[2:3], off
; GCN-SDAG-FAKE16-NEXT: global_load_u8 v10, v[0:1], off
; GCN-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x2
-; GCN-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v6
+; GCN-SDAG-FAKE16-NEXT: v_lshlrev_b16 v11, 8, v6
; GCN-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-FAKE16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
+; GCN-SDAG-FAKE16-NEXT: v_perm_b32 v7, v10, v7, 0xc0c0004
; GCN-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v6, v0
-; GCN-SDAG-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[8:9], v0, off
+; GCN-SDAG-FAKE16-NEXT: v_or_b32_e32 v6, v6, v11
+; GCN-SDAG-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v7
+; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[8:9], v6, off
; GCN-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-GISEL-LABEL: test_i8load_v4i8store:
@@ -35,32 +34,29 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-GISEL-NEXT: global_load_u8 v7, v[2:3], off
; GCN-GISEL-NEXT: global_load_u8 v10, v[4:5], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x1
-; GCN-GISEL-NEXT: s_wait_xcnt 0x2
-; GCN-GISEL-NEXT: v_lshl_or_b32 v0, v7, 8, v6
+; GCN-GISEL-NEXT: v_lshl_or_b32 v6, v7, 8, v6
; GCN-GISEL-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT: s_wait_xcnt 0x1
-; GCN-GISEL-NEXT: v_dual_lshlrev_b32 v1, 16, v10 :: v_dual_lshlrev_b32 v2, 24, v10
+; GCN-GISEL-NEXT: v_dual_lshlrev_b32 v7, 16, v10 :: v_dual_lshlrev_b32 v10, 24, v10
; GCN-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
-; GCN-GISEL-NEXT: global_store_b32 v[8:9], v0, off
+; GCN-GISEL-NEXT: v_or3_b32 v6, v6, v7, v10
+; GCN-GISEL-NEXT: global_store_b32 v[8:9], v6, off
; GCN-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-SDAG-REAL16-LABEL: test_i8load_v4i8store:
; GCN-SDAG-REAL16: ; %bb.0:
; GCN-SDAG-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GCN-SDAG-REAL16-NEXT: s_wait_kmcnt 0x0
-; GCN-SDAG-REAL16-NEXT: global_load_u8 v6, v[4:5], off
-; GCN-SDAG-REAL16-NEXT: global_load_u8 v7, v[2:3], off
-; GCN-SDAG-REAL16-NEXT: global_load_u8 v10, v[0:1], off
+; GCN-SDAG-REAL16-NEXT: global_load_u8 v7, v[4:5], off
+; GCN-SDAG-REAL16-NEXT: global_load_u8 v10, v[2:3], off
+; GCN-SDAG-REAL16-NEXT: global_load_u8 v11, v[0:1], off
; GCN-SDAG-REAL16-NEXT: s_wait_loadcnt 0x2
-; GCN-SDAG-REAL16-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-REAL16-NEXT: v_lshlrev_b16 v0.l, 8, v6.l
+; GCN-SDAG-REAL16-NEXT: v_lshlrev_b16 v6.l, 8, v7.l
; GCN-SDAG-REAL16-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-REAL16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
+; GCN-SDAG-REAL16-NEXT: v_perm_b32 v10, v11, v10, 0xc0c0004
; GCN-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-SDAG-REAL16-NEXT: v_or_b16 v0.l, v6.l, v0.l
-; GCN-SDAG-REAL16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v0, off
+; GCN-SDAG-REAL16-NEXT: v_or_b16 v6.l, v7.l, v6.l
+; GCN-SDAG-REAL16-NEXT: v_lshl_or_b32 v6, v6, 16, v10
+; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v6, off
; GCN-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
%a = load i8, ptr addrspace(1) %ptr_a
%b = load i8, ptr addrspace(1) %ptr_b
@@ -83,19 +79,18 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2
; GCN-SDAG-FAKE16-NEXT: global_load_b128 v[8:11], v[2:3], off
; GCN-SDAG-FAKE16-NEXT: v_mov_b64_e32 v[12:13], 0
; GCN-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-FAKE16-NEXT: s_wait_xcnt 0x1
-; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v1, v6, v10
-; GCN-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v3, v5, v9
-; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v5, v7, v11
+; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v5, v5, v9
+; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v9, v6, v10
+; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v14, v7, v11
; GCN-SDAG-FAKE16-NEXT: v_mov_b64_e32 v[6:7], 12
; GCN-SDAG-FAKE16-NEXT: v_mov_b64_e32 v[10:11], 8
-; GCN-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v2, v4, v8
+; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v4, v4, v8
+; GCN-SDAG-FAKE16-NEXT: s_wait_xcnt 0x1
+; GCN-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v9
; GCN-SDAG-FAKE16-NEXT: s_clause 0x2
-; GCN-SDAG-FAKE16-NEXT: global_store_b16 v[6:7], v5, off
-; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[10:11], v1, off
-; GCN-SDAG-FAKE16-NEXT: global_store_b64 v[12:13], v[2:3], off
+; GCN-SDAG-FAKE16-NEXT: global_store_b16 v[6:7], v14, off
+; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[10:11], v9, off
+; GCN-SDAG-FAKE16-NEXT: global_store_b64 v[12:13], v[4:5], off
; GCN-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-GISEL-FAKE16-LABEL: test_v7i16_load_store:
@@ -104,28 +99,28 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2
; GCN-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GCN-GISEL-FAKE16-NEXT: global_load_b128 v[4:7], v[0:1], off
; GCN-GISEL-FAKE16-NEXT: global_load_b128 v[8:11], v[2:3], off
-; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], 0
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[12:13], 2
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[14:15], 4
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[16:17], 6
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[18:19], 8
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[20:21], 10
-; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[22:23], 12
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[12:13], 0
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[14:15], 2
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[16:17], 4
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[18:19], 6
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[20:21], 8
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[22:23], 10
+; GCN-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[24:25], 12
; GCN-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v1, v6, v10
+; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v6, v6, v10
; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v4, v4, v8
; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v5, v5, v9
-; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v6, v7, v11
+; GCN-GISEL-FAKE16-NEXT: v_pk_add_u16 v7, v7, v11
; GCN-GISEL-FAKE16-NEXT: s_clause 0x6
-; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[2:3], v4, off
-; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[12:13], v4, off
-; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[14:15], v5, off
-; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[16:17], v5, off
-; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[18:19], v1, off
-; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[20:21], v1, off
-; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[22:23], v6, off
-; GCN-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[12:13], v4, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[14:15], v4, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[16:17], v5, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[18:19], v5, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[20:21], v6, off
+; GCN-GISEL-FAKE16-NEXT: global_store_d16_hi_b16 v[22:23], v6, off
+; GCN-GISEL-FAKE16-NEXT: global_store_b16 v[24:25], v7, off
+; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x8
+; GCN-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GCN-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-SDAG-REAL16-LABEL: test_v7i16_load_store:
@@ -134,19 +129,19 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2
; GCN-SDAG-REAL16-NEXT: s_wait_kmcnt 0x0
; GCN-SDAG-REAL16-NEXT: global_load_b128 v[4:7], v[0:1], off
; GCN-SDAG-REAL16-NEXT: global_load_b128 v[8:11], v[2:3], off
-; GCN-SDAG-REAL16-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[2:3], 12
-; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[12:13], 8
-; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[14:15], 0
+; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[12:13], 12
+; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[14:15], 8
+; GCN-SDAG-REAL16-NEXT: v_mov_b64_e32 v[16:17], 0
; GCN-SDAG-REAL16-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-REAL16-NEXT: v_pk_add_u16 v1, v7, v11
+; GCN-SDAG-REAL16-NEXT: v_pk_add_u16 v7, v7, v11
+; GCN-SDAG-REAL16-NEXT: s_wait_xcnt 0x1
; GCN-SDAG-REAL16-NEXT: v_pk_add_u16 v0, v6, v10
; GCN-SDAG-REAL16-NEXT: v_pk_add_u16 v5, v5, v9
; GCN-SDAG-REAL16-NEXT: v_pk_add_u16 v4, v4, v8
; GCN-SDAG-REAL16-NEXT: s_clause 0x2
-; GCN-SDAG-REAL16-NEXT: global_store_b16 v[2:3], v1, off
-; GCN-SDAG-REAL16-NEXT: global_store_b32 v[12:13], v0, off
-; GCN-SDAG-REAL16-NEXT: global_store_b64 v[14:15], v[4:5], off
+; GCN-SDAG-REAL16-NEXT: global_store_b16 v[12:13], v7, off
+; GCN-SDAG-REAL16-NEXT: global_store_b32 v[14:15], v0, off
+; GCN-SDAG-REAL16-NEXT: global_store_b64 v[16:17], v[4:5], off
; GCN-SDAG-REAL16-NEXT: s_wait_xcnt 0x1
; GCN-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GCN-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
@@ -157,28 +152,28 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2
; GCN-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
; GCN-GISEL-REAL16-NEXT: global_load_b128 v[4:7], v[0:1], off
; GCN-GISEL-REAL16-NEXT: global_load_b128 v[8:11], v[2:3], off
-; GCN-GISEL-REAL16-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[2:3], 0
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[12:13], 2
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[14:15], 4
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[16:17], 6
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[18:19], 8
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[20:21], 10
-; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[22:23], 12
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[12:13], 0
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[14:15], 2
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[16:17], 4
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[18:19], 6
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[20:21], 8
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[22:23], 10
+; GCN-GISEL-REAL16-NEXT: v_mov_b64_e32 v[24:25], 12
; GCN-GISEL-REAL16-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v1, v6, v10
+; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v6, v6, v10
; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v4, v4, v8
; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v5, v5, v9
-; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v6, v7, v11
+; GCN-GISEL-REAL16-NEXT: v_pk_add_u16 v7, v7, v11
; GCN-GISEL-REAL16-NEXT: s_clause 0x6
-; GCN-GISEL-REAL16-NEXT: global_store_b16 v[2:3], v4, off
-; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[12:13], v4, off
-; GCN-GISEL-REAL16-NEXT: global_store_b16 v[14:15], v5, off
-; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[16:17], v5, off
-; GCN-GISEL-REAL16-NEXT: global_store_b16 v[18:19], v1, off
-; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[20:21], v1, off
-; GCN-GISEL-REAL16-NEXT: global_store_b16 v[22:23], v6, off
-; GCN-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GCN-GISEL-REAL16-NEXT: global_store_b16 v[12:13], v4, off
+; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[14:15], v4, off
+; GCN-GISEL-REAL16-NEXT: global_store_b16 v[16:17], v5, off
+; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[18:19], v5, off
+; GCN-GISEL-REAL16-NEXT: global_store_b16 v[20:21], v6, off
+; GCN-GISEL-REAL16-NEXT: global_store_d16_hi_b16 v[22:23], v6, off
+; GCN-GISEL-REAL16-NEXT: global_store_b16 v[24:25], v7, off
+; GCN-GISEL-REAL16-NEXT: s_wait_xcnt 0x8
+; GCN-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GCN-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec1 = load <7 x i16>, ptr addrspace(1) %ptr1
%insert = insertelement <7 x i16> %vec1, i16 20, i32 4
@@ -325,64 +320,63 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-NEXT: global_load_b128 v[26:29], v[0:1], off offset:16
; GCN-SDAG-NEXT: global_load_b128 v[30:33], v[0:1], off
; GCN-SDAG-NEXT: global_load_b128 v[34:37], v[0:1], off offset:64
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[48:49], 48
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x70
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[50:51], 32
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[38:39], 0x60
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[64:65], 16
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[66:67], 0
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[52:53], 0x50
; GCN-SDAG-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, 0xc8
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[54:55], 64
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[0:1], 0x70
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[54:55], 48
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x60
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[64:65], 32
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[66:67], 16
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[68:69], 0
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[38:39], 0x50
+; GCN-SDAG-NEXT: v_dual_mov_b32 v49, 0 :: v_dual_mov_b32 v48, 0xc8
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[52:53], 64
; GCN-SDAG-NEXT: s_wait_loadcnt 0x7
-; GCN-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GCN-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x6
-; GCN-SDAG-NEXT: global_store_b128 v[38:39], v[10:13], off
+; GCN-SDAG-NEXT: global_store_b128 v[2:3], v[10:13], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x5
-; GCN-SDAG-NEXT: s_wait_xcnt 0x1
-; GCN-SDAG-NEXT: v_dual_mov_b32 v2, v16 :: v_dual_mov_b32 v3, v17
+; GCN-SDAG-NEXT: v_dual_mov_b32 v50, v16 :: v_dual_mov_b32 v51, v17
; GCN-SDAG-NEXT: s_wait_xcnt 0x0
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
-; GCN-SDAG-NEXT: s_wait_loadcnt 0x4
-; GCN-SDAG-NEXT: global_store_b128 v[48:49], v[18:21], off
-; GCN-SDAG-NEXT: s_wait_loadcnt 0x3
-; GCN-SDAG-NEXT: global_store_b128 v[50:51], v[22:25], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x2
-; GCN-SDAG-NEXT: global_store_b128 v[64:65], v[26:29], off
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[28:29], v[28:29]
+; GCN-SDAG-NEXT: s_clause 0x2
+; GCN-SDAG-NEXT: global_store_b128 v[54:55], v[18:21], off
+; GCN-SDAG-NEXT: global_store_b128 v[64:65], v[22:25], off
+; GCN-SDAG-NEXT: global_store_b128 v[66:67], v[26:29], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x1
-; GCN-SDAG-NEXT: global_store_b128 v[66:67], v[30:33], off
-; GCN-SDAG-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
+; GCN-SDAG-NEXT: global_store_b128 v[68:69], v[30:33], off
+; GCN-SDAG-NEXT: s_wait_xcnt 0x1
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[28:29], v[32:33], v[32:33]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[6:7]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[26:27], v[26:27]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[26:27], v[30:31], v[30:31]
; GCN-SDAG-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[50:51], v[36:37], v[36:37]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[48:49], v[34:35], v[34:35]
+; GCN-SDAG-NEXT: s_wait_xcnt 0x0
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[32:33], v[36:37], v[36:37]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[30:31], v[34:35], v[34:35]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[14:15], 0xc8, v[14:15]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[28:29], v[28:29], v[28:29]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[26:27], v[26:27], v[26:27]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[30:31], v[30:31], v[30:31]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[20:21], v[20:21], v[20:21]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[18:19], v[18:19], v[18:19]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[24:25], 0x64, v[24:25]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[22:23], v[22:23], v[22:23]
; GCN-SDAG-NEXT: s_clause 0x1
-; GCN-SDAG-NEXT: global_store_b128 v[52:53], v[0:3], off
-; GCN-SDAG-NEXT: global_store_b128 v[54:55], v[34:37], off
+; GCN-SDAG-NEXT: global_store_b128 v[38:39], v[48:51], off
+; GCN-SDAG-NEXT: global_store_b128 v[52:53], v[34:37], off
; GCN-SDAG-NEXT: s_clause 0x7
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[10:13], off offset:96
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[6:9], off offset:112
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[48:51], off offset:64
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[30:33], off offset:64
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[14:17], off offset:80
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[22:25], off offset:32
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[18:21], off offset:48
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[30:33], off
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[26:29], off offset:16
-; GCN-SDAG-NEXT: s_wait_xcnt 0x9
-; GCN-SDAG-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[26:29], off
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16
+; GCN-SDAG-NEXT: s_wait_xcnt 0x0
+; GCN-SDAG-NEXT: v_dual_mov_b32 v0, v28 :: v_dual_mov_b32 v1, v29
; GCN-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-GISEL-LABEL: test_v16i64_load_store:
@@ -398,44 +392,43 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-GISEL-NEXT: global_load_b128 v[26:29], v[0:1], off offset:96
; GCN-GISEL-NEXT: global_load_b128 v[30:33], v[0:1], off offset:112
; GCN-GISEL-NEXT: global_load_b128 v[34:37], v[0:1], off offset:64
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[38:39], 0
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[48:49], 16
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[50:51], 32
+; GCN-GISEL-NEXT: s_wait_xcnt 0x0
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[0:1], 0
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[2:3], 16
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[38:39], 32
; GCN-GISEL-NEXT: v_mov_b64_e32 v[52:53], 48
; GCN-GISEL-NEXT: v_mov_b64_e32 v[66:67], 0x60
; GCN-GISEL-NEXT: v_mov_b64_e32 v[68:69], 0x70
; GCN-GISEL-NEXT: v_mov_b64_e32 v[54:55], 64
-; GCN-GISEL-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[0:1], 0xc8
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[48:49], 0xc8
; GCN-GISEL-NEXT: v_mov_b64_e32 v[64:65], 0x50
; GCN-GISEL-NEXT: s_wait_loadcnt 0x6
-; GCN-GISEL-NEXT: global_store_b128 v[38:39], v[10:13], off
+; GCN-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x5
-; GCN-GISEL-NEXT: global_store_b128 v[48:49], v[14:17], off
+; GCN-GISEL-NEXT: global_store_b128 v[2:3], v[14:17], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x4
-; GCN-GISEL-NEXT: global_store_b128 v[50:51], v[18:21], off
+; GCN-GISEL-NEXT: global_store_b128 v[38:39], v[18:21], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x3
; GCN-GISEL-NEXT: global_store_b128 v[52:53], v[22:25], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x2
; GCN-GISEL-NEXT: global_store_b128 v[66:67], v[26:29], off
; GCN-GISEL-NEXT: s_wait_loadcnt 0x1
; GCN-GISEL-NEXT: global_store_b128 v[68:69], v[30:33], off
-; GCN-GISEL-NEXT: s_wait_xcnt 0x5
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[8:9]
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
; GCN-GISEL-NEXT: s_wait_xcnt 0x4
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[14:15], v[14:15], v[14:15]
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], v[12:13], v[12:13]
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[50:51], v[8:9]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[10:11], v[10:11]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[10:11], v[14:15], v[14:15]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[12:13], v[16:17], v[16:17]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[14:15], v[18:19], v[18:19]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[16:17], 0x64, v[20:21]
; GCN-GISEL-NEXT: s_wait_xcnt 0x3
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[18:19], v[18:19], v[18:19]
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[20:21], 0x64, v[20:21]
-; GCN-GISEL-NEXT: s_wait_xcnt 0x2
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[22:23], v[22:23], v[22:23]
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[24:25], v[24:25], v[24:25]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[18:19], v[22:23], v[22:23]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[20:21], v[24:25], v[24:25]
; GCN-GISEL-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[48:49], v[34:35], v[34:35]
-; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[50:51], v[36:37], v[36:37]
+; GCN-GISEL-NEXT: s_wait_xcnt 0x2
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[22:23], v[34:35], v[34:35]
+; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[24:25], v[36:37], v[36:37]
; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[6:7], 0xc8, v[6:7]
; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9]
; GCN-GISEL-NEXT: s_wait_xcnt 0x1
@@ -446,18 +439,18 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
; GCN-GISEL-NEXT: s_clause 0x1
; GCN-GISEL-NEXT: global_store_b128 v[54:55], v[34:37], off
-; GCN-GISEL-NEXT: global_store_b128 v[64:65], v[0:3], off
+; GCN-GISEL-NEXT: global_store_b128 v[64:65], v[48:51], off
; GCN-GISEL-NEXT: s_clause 0x7
-; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off
-; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[14:17], off offset:16
-; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[18:21], off offset:32
-; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[22:25], off offset:48
-; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[48:51], off offset:64
+; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[10:13], off offset:16
+; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[14:17], off offset:32
+; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[18:21], off offset:48
+; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[22:25], off offset:64
; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[6:9], off offset:80
; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[26:29], off offset:96
; GCN-GISEL-NEXT: global_store_b128 v[4:5], v[30:33], off offset:112
-; GCN-GISEL-NEXT: s_wait_xcnt 0x8
-; GCN-GISEL-NEXT: v_dual_mov_b32 v0, v12 :: v_dual_mov_b32 v1, v13
+; GCN-GISEL-NEXT: s_wait_xcnt 0x7
+; GCN-GISEL-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GCN-GISEL-NEXT: s_set_pc_i64 s[30:31]
%a = load <16 x i64>, ptr addrspace(1) %ptr_a, align 4
%in_a = insertelement <16 x i64> %a, i64 100, i32 5
@@ -479,17 +472,16 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1,
; GCN-SDAG-NEXT: v_nop
; GCN-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
-; GCN-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GCN-SDAG-NEXT: v_and_b32_e32 v14, 0x3ff, v0
; GCN-SDAG-NEXT: s_wait_xcnt 0x0
; GCN-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
+; GCN-SDAG-NEXT: v_mov_b64_e32 v[8:9], 12
; GCN-SDAG-NEXT: v_mov_b64_e32 v[10:11], 8
; GCN-SDAG-NEXT: v_mov_b64_e32 v[12:13], 0
; GCN-SDAG-NEXT: s_wait_kmcnt 0x0
; GCN-SDAG-NEXT: s_clause 0x1
-; GCN-SDAG-NEXT: global_load_b128 v[0:3], v8, s[0:1] scale_offset
-; GCN-SDAG-NEXT: global_load_b128 v[4:7], v8, s[2:3] scale_offset
-; GCN-SDAG-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-NEXT: v_mov_b64_e32 v[8:9], 12
+; GCN-SDAG-NEXT: global_load_b128 v[0:3], v14, s[0:1] scale_offset
+; GCN-SDAG-NEXT: global_load_b128 v[4:7], v14, s[2:3] scale_offset
; GCN-SDAG-NEXT: s_wait_loadcnt 0x0
; GCN-SDAG-NEXT: v_pk_add_u16 v3, v3, v7
; GCN-SDAG-NEXT: v_pk_add_u16 v2, v2, v6
@@ -510,9 +502,10 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1,
; GCN-GISEL-NEXT: v_nop
; GCN-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
-; GCN-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GCN-GISEL-NEXT: v_and_b32_e32 v22, 0x3ff, v0
; GCN-GISEL-NEXT: s_wait_xcnt 0x0
; GCN-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
+; GCN-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0
; GCN-GISEL-NEXT: v_mov_b64_e32 v[10:11], 2
; GCN-GISEL-NEXT: v_mov_b64_e32 v[12:13], 4
; GCN-GISEL-NEXT: v_mov_b64_e32 v[14:15], 6
@@ -521,10 +514,8 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1,
; GCN-GISEL-NEXT: v_mov_b64_e32 v[20:21], 12
; GCN-GISEL-NEXT: s_wait_kmcnt 0x0
; GCN-GISEL-NEXT: s_clause 0x1
-; GCN-GISEL-NEXT: global_load_b128 v[0:3], v8, s[0:1] scale_offset
-; GCN-GISEL-NEXT: global_load_b128 v[4:7], v8, s[2:3] scale_offset
-; GCN-GISEL-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0
+; GCN-GISEL-NEXT: global_load_b128 v[0:3], v22, s[0:1] scale_offset
+; GCN-GISEL-NEXT: global_load_b128 v[4:7], v22, s[2:3] scale_offset
; GCN-GISEL-NEXT: s_wait_loadcnt 0x0
; GCN-GISEL-NEXT: v_pk_add_u16 v0, v0, v4
; GCN-GISEL-NEXT: v_pk_add_u16 v1, v1, v5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 0413dfa292d11d..fff115db2d838a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -6706,12 +6706,11 @@ define <4 x float> @global_load_f32_natural_addressing(ptr addrspace(1) %sbase,
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -6723,14 +6722,14 @@ define <4 x float> @global_load_f32_natural_addressing(ptr addrspace(1) %sbase,
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1310-SDAG-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -6791,12 +6790,11 @@ define <4 x float> @global_load_f32_natural_addressing(ptr addrspace(1) %sbase,
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-ISEL-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -6808,14 +6806,14 @@ define <4 x float> @global_load_f32_natural_addressing(ptr addrspace(1) %sbase,
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX1310-ISEL-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v5, 0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -6878,12 +6876,11 @@ define <4 x float> @global_load_f32_natural_addressing_immoffset(ptr addrspace(1
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -7044,10 +7041,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range(ptr addrspace(1) %sbase, ptr
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_lshlrev_b32 v4, 2, v2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -7061,9 +7057,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range(ptr addrspace(1) %sbase, ptr
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v2, v[2:3], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -7125,10 +7121,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range(ptr addrspace(1) %sbase, ptr
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
@@ -7143,9 +7138,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range(ptr addrspace(1) %sbase, ptr
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: global_load_b32 v2, v[2:3], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
@@ -7215,10 +7210,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range_imm_offset(ptr addrspace(1)
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_lshlrev_b32 v4, 2, v2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -7232,9 +7226,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range_imm_offset(ptr addrspace(1)
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v2, v[2:3], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -7296,10 +7290,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range_imm_offset(ptr addrspace(1)
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: global_load_b32 v2, v[2:3], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
@@ -7314,9 +7307,9 @@ define <4 x float> @global_load_f32_zext_vgpr_range_imm_offset(ptr addrspace(1)
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1310-ISEL-NEXT: global_load_b32 v2, v[2:3], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v2
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
@@ -7388,12 +7381,11 @@ define <4 x float> @global_load_f32_zext_vgpr_range_too_large(ptr addrspace(1) %
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -7405,14 +7397,14 @@ define <4 x float> @global_load_f32_zext_vgpr_range_too_large(ptr addrspace(1) %
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1310-SDAG-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -7473,12 +7465,11 @@ define <4 x float> @global_load_f32_zext_vgpr_range_too_large(ptr addrspace(1) %
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-ISEL-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 2, v[0:1]
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -7490,14 +7481,14 @@ define <4 x float> @global_load_f32_zext_vgpr_range_too_large(ptr addrspace(1) %
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v3, 0
+; GFX1310-ISEL-NEXT: global_load_b32 v4, v[2:3], off
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v5, 0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[4:5], 2, v[4:5]
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -15415,13 +15406,12 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 2, s[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15432,15 +15422,15 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-SDAG-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
+; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v2, vcc_lo, s0, v2
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v3, vcc_lo
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15505,13 +15495,12 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 2, s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15522,16 +15511,16 @@ define <4 x float> @global_load_saddr_f32_natural_addressing(ptr addrspace(1) in
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v3, s1
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v5, s1
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX1310-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, v5, v3, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
@@ -15584,13 +15573,12 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15603,10 +15591,10 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_u32 v2, s0, s0, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off offset:128 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15650,14 +15638,13 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
+; GFX1250-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off offset:128
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15668,13 +15655,13 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off offset:128 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
@@ -15734,11 +15721,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15751,11 +15737,11 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v2, s0, s0, v2
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15803,15 +15789,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15822,14 +15807,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !4, !noundef !{}
@@ -15888,11 +15873,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off offset:400 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15905,11 +15889,11 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v2, s0, s0, v2
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off offset:400 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15957,15 +15941,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off offset:400 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15976,14 +15959,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off offset:400 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !4, !noundef !{}
@@ -16056,13 +16039,12 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 2, s[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -16073,15 +16055,15 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1310-SDAG-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
+; GFX1310-SDAG-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v2, vcc_lo, s0, v2
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v3, null, s1, v3, vcc_lo
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -16146,13 +16128,12 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-ISEL-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 2, s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -16163,16 +16144,16 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_too_large(ptr addrspac
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v3, s1
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v5, s1
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1]
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1310-ISEL-NEXT: v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX1310-ISEL-NEXT: v_add_co_u32 v2, vcc_lo, v4, v2
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v3, null, v5, v3, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[2:3], off
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !5, !noundef !{}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index f0909a0cfb30b3..17a05f4c861027 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -439,12 +439,12 @@ define void @global_store_i8_zext_vgpr(ptr addrspace(1) %sbase, ptr addrspace(1)
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v8, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[4:7], off
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -725,12 +725,12 @@ define void @global_store_i8_zext_vgpr_offset_2047(ptr addrspace(1) %sbase, ptr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v8, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[4:7], off offset:2047 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -877,12 +877,12 @@ define void @global_store_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) %sbase, p
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v2, v[2:3], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v8, v[2:3], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[4:7], off offset:-2048 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -1024,13 +1024,13 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], s[0:1], v[6:7]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1042,10 +1042,10 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_u32 v6, s0, s0, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v7, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1086,14 +1086,14 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1103,13 +1103,13 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1310-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -1278,13 +1278,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], s[0:1], v[6:7]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1296,10 +1296,10 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_u32 v6, s0, s0, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v7, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1340,14 +1340,14 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1357,13 +1357,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -1413,13 +1413,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], s[0:1], v[6:7]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1431,10 +1431,10 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_u32 v6, s0, s0, v0
; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v7, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[6:7], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1475,14 +1475,14 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1492,13 +1492,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
-; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[6:7], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
index d71cfa94c7f96f..2038f661e65ce0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.pk.add.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12PLUS %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX12PLUS %s
-; RUN: llc -mtriple=amdgpu13.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12PLUS %s
-; RUN: llc -mtriple=amdgpu13.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX12PLUS %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12 %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX12 %s
+; RUN: llc -mtriple=amdgpu13.10 -global-isel=0 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX13 %s
+; RUN: llc -mtriple=amdgpu13.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX12PLUS,GFX13 %s
define amdgpu_ps float @atomic_pk_add_f16_1d_v2(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {
; GFX12PLUS-LABEL: atomic_pk_add_f16_1d_v2:
@@ -64,15 +64,25 @@ main_body:
}
define amdgpu_ps float @atomic_pk_add_bf16_1d_v2(<8 x i32> inreg %rsrc, <2 x bfloat> %data, i32 %s) {
-; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v2:
-; GFX12PLUS: ; %bb.0: ; %main_body
-; GFX12PLUS-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
-; GFX12PLUS-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
-; GFX12PLUS-NEXT: s_wait_loadcnt 0x0
-; GFX12PLUS-NEXT: flat_store_b32 v[1:2], v0
-; GFX12PLUS-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX12PLUS-NEXT: s_wait_dscnt 0x0
-; GFX12PLUS-NEXT: ; return to shader part epilog
+; GFX12-LABEL: atomic_pk_add_bf16_1d_v2:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: flat_store_b32 v[1:2], v0
+; GFX12-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX13-LABEL: atomic_pk_add_bf16_1d_v2:
+; GFX13: ; %bb.0: ; %main_body
+; GFX13-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
+; GFX13-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX13-NEXT: s_wait_loadcnt 0x0
+; GFX13-NEXT: flat_store_b32 v[2:3], v0
+; GFX13-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX13-NEXT: s_wait_dscnt 0x0
+; GFX13-NEXT: ; return to shader part epilog
main_body:
%out = call <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
store <2 x bfloat> %out, ptr null
@@ -91,15 +101,25 @@ main_body:
}
define amdgpu_ps float @atomic_pk_add_bf16_1d_v4(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {
-; GFX12PLUS-LABEL: atomic_pk_add_bf16_1d_v4:
-; GFX12PLUS: ; %bb.0: ; %main_body
-; GFX12PLUS-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
-; GFX12PLUS-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX12PLUS-NEXT: s_wait_loadcnt 0x0
-; GFX12PLUS-NEXT: flat_store_b64 v[2:3], v[0:1]
-; GFX12PLUS-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX12PLUS-NEXT: s_wait_dscnt 0x0
-; GFX12PLUS-NEXT: ; return to shader part epilog
+; GFX12-LABEL: atomic_pk_add_bf16_1d_v4:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX13-LABEL: atomic_pk_add_bf16_1d_v4:
+; GFX13: ; %bb.0: ; %main_body
+; GFX13-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN
+; GFX13-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, 0
+; GFX13-NEXT: s_wait_loadcnt 0x0
+; GFX13-NEXT: flat_store_b64 v[3:4], v[0:1]
+; GFX13-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX13-NEXT: s_wait_dscnt 0x0
+; GFX13-NEXT: ; return to shader part epilog
main_body:
%out = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
store <4 x bfloat> %out, ptr null
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
index c226ad5679140c..964e8ca9c447ca 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
@@ -4083,13 +4083,21 @@ define amdgpu_ps void @image_store_wait(<8 x i32> inreg %arg, <8 x i32> inreg %a
; GFX10PLUS-NEXT: image_store v[0:3], v4, s[16:23] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
; GFX10PLUS-NEXT: s_endpgm
;
-; GFX12PLUS-LABEL: image_store_wait:
-; GFX12PLUS: ; %bb.0: ; %main_body
-; GFX12PLUS-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX12PLUS-NEXT: image_load v[0:3], v4, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX12PLUS-NEXT: s_wait_loadcnt 0x0
-; GFX12PLUS-NEXT: image_store v[0:3], v4, s[16:23] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX12PLUS-NEXT: s_endpgm
+; GFX12-LABEL: image_store_wait:
+; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: image_load v[0:3], v4, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: image_store v[0:3], v4, s[16:23] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX12-NEXT: s_endpgm
+;
+; GFX13-LABEL: image_store_wait:
+; GFX13: ; %bb.0: ; %main_body
+; GFX13-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX13-NEXT: image_load v[5:8], v4, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX13-NEXT: s_wait_loadcnt 0x0
+; GFX13-NEXT: image_store v[5:8], v4, s[16:23] dmask:0xf dim:SQ_RSRC_IMG_1D
+; GFX13-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %arg3, i32 15, i32 %arg4, <8 x i32> %arg, i32 0, i32 0)
%data = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %arg4, <8 x i32> %arg1, i32 0, i32 0)
@@ -4162,15 +4170,25 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, ptr addrspace(3) %
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
-; GFX12PLUS-LABEL: image_load_mmo:
-; GFX12PLUS: ; %bb.0:
-; GFX12PLUS-NEXT: image_load v1, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
-; GFX12PLUS-NEXT: v_mov_b32_e32 v2, 0
-; GFX12PLUS-NEXT: ds_store_2addr_b32 v0, v2, v2 offset1:4
-; GFX12PLUS-NEXT: s_wait_loadcnt 0x0
-; GFX12PLUS-NEXT: v_mov_b32_e32 v0, v1
-; GFX12PLUS-NEXT: s_wait_dscnt 0x0
-; GFX12PLUS-NEXT: ; return to shader part epilog
+; GFX12-LABEL: image_load_mmo:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: image_load v1, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: ds_store_2addr_b32 v0, v2, v2 offset1:4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: ; return to shader part epilog
+;
+; GFX13-LABEL: image_load_mmo:
+; GFX13: ; %bb.0:
+; GFX13-NEXT: image_load v1, [v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX13-NEXT: v_mov_b32_e32 v3, 0
+; GFX13-NEXT: ds_store_2addr_b32 v0, v3, v3 offset1:4
+; GFX13-NEXT: s_wait_loadcnt 0x0
+; GFX13-NEXT: v_mov_b32_e32 v0, v1
+; GFX13-NEXT: s_wait_dscnt 0x0
+; GFX13-NEXT: ; return to shader part epilog
store float 0.000000e+00, ptr addrspace(3) %lds
%c0 = extractelement <2 x i32> %c, i32 0
%c1 = extractelement <2 x i32> %c, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index c9a6e388e4aaa5..45bff4c0e55e7a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -70,11 +70,11 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[0:1] scale_offset scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v1
+; GFX1250-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, 0x4000000, v0
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
+; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, 0x4000000, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v1, off
; GFX1250-NEXT: s_endpgm
;
; CI-GISEL-LABEL: is_private_vgpr:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
index 273b534f600080..2c11e090809a13 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -105,8 +105,8 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3] scale_offset scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s1, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v1, off
; GFX1250-NEXT: s_endpgm
;
; CI-GISEL-LABEL: is_local_vgpr:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index 487e260b31d83c..11a490b5711101 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -952,20 +952,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 2
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v37, s19
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
@@ -1055,20 +1054,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v37, s19
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
@@ -1221,20 +1219,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: s_nop 2
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v37, s19
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
@@ -1324,20 +1321,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: s_nop 2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v37, s19
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
+; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 42cd877c422617..fca982999e093b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -1420,20 +1420,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
; SDAG-NEXT: v_mov_b32_e32 v44, s22
; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-NEXT: v_mov_b32_e32 v37, s19
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
@@ -1523,20 +1522,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
; HEURRC-NEXT: v_mov_b32_e32 v44, s22
; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 2
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v34, s16
+; HEURRC-NEXT: v_mov_b32_e32 v35, s17
+; HEURRC-NEXT: v_mov_b32_e32 v36, s18
+; HEURRC-NEXT: v_mov_b32_e32 v37, s19
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
+; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -1580,20 +1578,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 2
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s19
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
@@ -1762,20 +1759,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: v_mov_b32_e32 v44, s22
; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 2
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-NEXT: v_mov_b32_e32 v37, s19
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
@@ -1865,20 +1861,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: v_mov_b32_e32 v44, s22
; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 2
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v34, s16
+; HEURRC-NEXT: v_mov_b32_e32 v35, s17
+; HEURRC-NEXT: v_mov_b32_e32 v36, s18
+; HEURRC-NEXT: v_mov_b32_e32 v37, s19
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
+; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -1922,20 +1917,19 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 2
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s19
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
@@ -3974,25 +3968,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-NEXT: s_nop 6
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v34, s20
+; SDAG-NEXT: v_mov_b32_e32 v35, s21
+; SDAG-NEXT: v_mov_b32_e32 v36, s22
+; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-NEXT: v_mov_b32_e32 v37, s19
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
@@ -4084,25 +4076,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; HEURRC-NEXT: s_nop 6
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v34, s20
+; HEURRC-NEXT: v_mov_b32_e32 v35, s21
+; HEURRC-NEXT: v_mov_b32_e32 v36, s22
+; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
+; HEURRC-NEXT: v_mov_b32_e32 v34, s16
+; HEURRC-NEXT: v_mov_b32_e32 v35, s17
+; HEURRC-NEXT: v_mov_b32_e32 v36, s18
+; HEURRC-NEXT: v_mov_b32_e32 v37, s19
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -4148,25 +4138,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPRRC-NEXT: s_nop 6
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s19
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
@@ -4351,25 +4339,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-NEXT: s_nop 6
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v34, s20
+; SDAG-NEXT: v_mov_b32_e32 v35, s21
+; SDAG-NEXT: v_mov_b32_e32 v36, s22
+; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v34, s16
+; SDAG-NEXT: v_mov_b32_e32 v35, s17
+; SDAG-NEXT: v_mov_b32_e32 v36, s18
+; SDAG-NEXT: v_mov_b32_e32 v37, s19
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
@@ -4461,25 +4447,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; HEURRC-NEXT: s_nop 6
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mov_b32_e32 v16, s16
-; HEURRC-NEXT: v_mov_b32_e32 v17, s17
-; HEURRC-NEXT: v_mov_b32_e32 v18, s18
-; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v34, s20
+; HEURRC-NEXT: v_mov_b32_e32 v35, s21
+; HEURRC-NEXT: v_mov_b32_e32 v36, s22
+; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
+; HEURRC-NEXT: v_mov_b32_e32 v34, s16
+; HEURRC-NEXT: v_mov_b32_e32 v35, s17
+; HEURRC-NEXT: v_mov_b32_e32 v36, s18
+; HEURRC-NEXT: v_mov_b32_e32 v37, s19
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
+; HEURRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
@@ -4525,25 +4509,23 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPRRC-NEXT: s_nop 6
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
-; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s19
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
+; VGPRRC-NEXT: global_store_dwordx4 v32, v[34:37], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index aba29a9a1ec1aa..5539c7c168a921 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -8244,34 +8244,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; SDAG-NEXT: v_mov_b32_e32 v34, s20
+; SDAG-NEXT: v_mov_b32_e32 v35, s21
+; SDAG-NEXT: v_mov_b32_e32 v36, s22
+; SDAG-NEXT: v_mov_b32_e32 v37, s23
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; SDAG-NEXT: s_nop 12
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[34:37], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s10
-; SDAG-NEXT: v_mov_b32_e32 v23, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v38, s18
+; SDAG-NEXT: v_mov_b32_e32 v39, s19
+; SDAG-NEXT: v_mov_b32_e32 v36, s16
+; SDAG-NEXT: v_mov_b32_e32 v37, s17
+; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[36:39], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
; SDAG-NEXT: v_mov_b32_e32 v20, s8
+; SDAG-NEXT: v_mov_b32_e32 v36, s12
+; SDAG-NEXT: v_mov_b32_e32 v37, s13
+; SDAG-NEXT: v_mov_b32_e32 v38, s14
+; SDAG-NEXT: v_mov_b32_e32 v39, s15
; SDAG-NEXT: v_mov_b32_e32 v21, s9
+; SDAG-NEXT: v_mov_b32_e32 v22, s10
+; SDAG-NEXT: v_mov_b32_e32 v23, s11
; SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; SDAG-NEXT: global_store_dwordx4 v[16:17], v[36:39], off sc0 sc1
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
@@ -8288,6 +8287,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
+; GISEL-NEXT: v_mov_b64_e32 v[48:49], 0
+; GISEL-NEXT: v_mov_b64_e32 v[50:51], 16
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[38:39]
@@ -8307,26 +8308,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; GISEL-NEXT: v_mov_b64_e32 v[32:33], 0
-; GISEL-NEXT: v_mov_b64_e32 v[34:35], 16
-; GISEL-NEXT: v_mov_b64_e32 v[36:37], 32
-; GISEL-NEXT: v_mov_b64_e32 v[38:39], 48
-; GISEL-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
+; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[8:9]
+; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[14:15]
+; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[12:13]
+; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[18:19]
+; GISEL-NEXT: s_nop 9
+; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
+; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
+; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[16:17]
+; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[22:23]
+; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[20:21]
+; GISEL-NEXT: global_store_dwordx4 v[48:49], v[32:35], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[34:35], v[20:23], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[50:51], v[36:39], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[36:37], v[24:27], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[16:17], v[40:43], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[38:39], v[28:31], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[18:19], v[44:47], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_nop 7
-; GISEL-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[50:51], v[4:7], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[36:37], v[8:11], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: global_store_dwordx4 v[38:39], v[12:15], off sc0 sc1
+; GISEL-NEXT: global_store_dwordx4 v[18:19], v[12:15], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
@@ -8362,34 +8369,33 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v34, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v35, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
-; AGPR-SDAG-NEXT: s_nop 12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
-; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[16:19], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[32:33], v[34:37], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v38, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v39, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s17
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[36:39], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 16
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v38, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v39, s15
; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s11
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 0
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[36:39], off sc0 sc1
+; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
@@ -8406,6 +8412,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-GISEL: ; %bb.0:
; AGPR-GISEL-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; AGPR-GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[48:49], 0
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[50:51], 16
; AGPR-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[36:37]
; AGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[38:39]
@@ -8425,26 +8433,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-GISEL-NEXT: s_nop 1
; AGPR-GISEL-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], 0
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], 16
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[36:37], 32
-; AGPR-GISEL-NEXT: v_mov_b64_e32 v[38:39], 48
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[32:33], v[16:19], off sc0 sc1
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[34:35], s[10:11]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[32:33], s[8:9]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[38:39], s[14:15]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[36:37], s[12:13]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[42:43], s[18:19]
+; AGPR-GISEL-NEXT: s_nop 9
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[40:41], s[16:17]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[46:47], s[22:23]
+; AGPR-GISEL-NEXT: v_mov_b64_e32 v[44:45], s[20:21]
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[48:49], v[32:35], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[34:35], v[20:23], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[50:51], v[36:39], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[36:37], v[24:27], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], v[40:43], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[38:39], v[28:31], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], v[44:47], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: s_nop 7
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[50:51], v[4:7], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[36:37], v[8:11], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[16:17], v[8:11], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
-; AGPR-GISEL-NEXT: global_store_dwordx4 v[38:39], v[12:15], off sc0 sc1
+; AGPR-GISEL-NEXT: global_store_dwordx4 v[18:19], v[12:15], off sc0 sc1
; AGPR-GISEL-NEXT: s_waitcnt vmcnt(0)
; AGPR-GISEL-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 0, i32 2, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.ll
index b5a30c9511606f..8a912053aac012 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load.ll
@@ -796,18 +796,19 @@ define amdgpu_kernel void @ptr_s_buffer_load_v8i32_divergent_offset_glc(ptr addr
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x10 nv
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_bitop2_b32 v8, s6, v0 bitop3:0x40
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: buffer_load_b128 v[0:3], v8, s[0:3], null offen offset:16 th:TH_LOAD_NT nv
; GFX1250-SDAG-NEXT: buffer_load_b128 v[4:7], v8, s[0:3], null offen th:TH_LOAD_NT nv
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x1
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v9, v[0:3], s[0:1] offset:16
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v9, v[4:7], s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
@@ -816,17 +817,18 @@ define amdgpu_kernel void @ptr_s_buffer_load_v8i32_divergent_offset_glc(ptr addr
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x10 nv
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1250-GISEL-NEXT: s_mov_b32 s6, 0x3ff
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_bitop2_b32 v8, s6, v0 bitop3:0x40
; GFX1250-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_clause 0x1
; GFX1250-GISEL-NEXT: buffer_load_b128 v[0:3], v8, s[0:3], null offen th:TH_LOAD_NT nv
; GFX1250-GISEL-NEXT: buffer_load_b128 v[4:7], v8, s[0:3], null offen offset:16 th:TH_LOAD_NT nv
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1250-GISEL-NEXT: global_store_b128 v8, v[0:3], s[4:5]
+; GFX1250-GISEL-NEXT: global_store_b128 v9, v[0:3], s[4:5]
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: global_store_b128 v8, v[4:7], s[4:5] offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v9, v[4:7], s[4:5] offset:16
; GFX1250-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%load = call <8 x i32> @llvm.amdgcn.ptr.s.buffer.load.v8i32(ptr addrspace(8) %rsrc, i32 %tid, i32 1), !invariant.load !0
@@ -1048,7 +1050,9 @@ define amdgpu_kernel void @ptr_s_buffer_load_v16i32_divergent_offset(ptr addrspa
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x10 nv
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v16, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s6, 0x3ff
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_bitop2_b32 v16, s6, v0 bitop3:0x40
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_clause 0x3
; GFX1250-SDAG-NEXT: buffer_load_b128 v[0:3], v16, s[0:3], null offen offset:48 nv
@@ -1056,16 +1060,15 @@ define amdgpu_kernel void @ptr_s_buffer_load_v16i32_divergent_offset(ptr addrspa
; GFX1250-SDAG-NEXT: buffer_load_b128 v[8:11], v16, s[0:3], null offen offset:16 nv
; GFX1250-SDAG-NEXT: buffer_load_b128 v[12:15], v16, s[0:3], null offen nv
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x3
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v17, v[0:3], s[0:1] offset:48
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v17, v[4:7], s[0:1] offset:32
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x1
-; GFX1250-SDAG-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v17, v[8:11], s[0:1] offset:16
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v17, v[12:15], s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
@@ -1074,7 +1077,9 @@ define amdgpu_kernel void @ptr_s_buffer_load_v16i32_divergent_offset(ptr addrspa
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x10 nv
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v16, 0x3ff, v0
+; GFX1250-GISEL-NEXT: s_mov_b32 s6, 0x3ff
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_bitop2_b32 v16, s6, v0 bitop3:0x40
; GFX1250-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_clause 0x3
@@ -1082,15 +1087,14 @@ define amdgpu_kernel void @ptr_s_buffer_load_v16i32_divergent_offset(ptr addrspa
; GFX1250-GISEL-NEXT: buffer_load_b128 v[4:7], v16, s[0:3], null offen offset:16 nv
; GFX1250-GISEL-NEXT: buffer_load_b128 v[8:11], v16, s[0:3], null offen offset:32 nv
; GFX1250-GISEL-NEXT: buffer_load_b128 v[12:15], v16, s[0:3], null offen offset:48 nv
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x3
-; GFX1250-GISEL-NEXT: global_store_b128 v16, v[0:3], s[4:5]
+; GFX1250-GISEL-NEXT: global_store_b128 v17, v[0:3], s[4:5]
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x2
-; GFX1250-GISEL-NEXT: global_store_b128 v16, v[4:7], s[4:5] offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v17, v[4:7], s[4:5] offset:16
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x1
-; GFX1250-GISEL-NEXT: global_store_b128 v16, v[8:11], s[4:5] offset:32
+; GFX1250-GISEL-NEXT: global_store_b128 v17, v[8:11], s[4:5] offset:32
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: global_store_b128 v16, v[12:15], s[4:5] offset:48
+; GFX1250-GISEL-NEXT: global_store_b128 v17, v[12:15], s[4:5] offset:48
; GFX1250-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%load = call <16 x i32> @llvm.amdgcn.ptr.s.buffer.load.v16i32(ptr addrspace(8) %rsrc, i32 %tid, i32 0), !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index 5e79b67fb5499b..9e754db5cf1f5d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -148,9 +148,9 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32,
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null offen
-; GFX1250-NEXT: buffer_load_b128 v[0:3], v5, s[0:3], null offen
+; GFX1250-NEXT: buffer_load_b128 v[8:11], v5, s[0:3], null offen
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_store_b128 v[0:3], v6, s[0:3], null offen
+; GFX1250-NEXT: buffer_store_b128 v[8:11], v6, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
index f677f5dfe06cae..6840455e97652d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
@@ -94,12 +94,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; SDAG-TRUE16-NEXT: s_mov_b32 s4, 0x3ff
; SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v1, s4, v0 bitop3:0x40
; SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; SDAG-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scale_offset
; SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; SDAG-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, v0.l
-; SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; SDAG-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; SDAG-TRUE16-NEXT: s_endpgm
;
; SDAG-FAKE16-LABEL: rcp_bf16_global_load:
@@ -112,12 +112,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; SDAG-FAKE16-NEXT: s_mov_b32 s4, 0x3ff
; SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; SDAG-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
; SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v0, v0
-; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v1, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v2, v1, s[0:1]
; SDAG-FAKE16-NEXT: s_endpgm
;
; GI-TRUE16-LABEL: rcp_bf16_global_load:
@@ -130,12 +130,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; GI-TRUE16-NEXT: s_wait_xcnt 0x0
; GI-TRUE16-NEXT: s_mov_b32 s4, 0x3ff
; GI-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GI-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GI-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v1, s4, v0 bitop3:0x40
; GI-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GI-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
+; GI-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scale_offset
; GI-TRUE16-NEXT: s_wait_loadcnt 0x0
; GI-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, v0.l
-; GI-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GI-TRUE16-NEXT: s_endpgm
;
; GI-FAKE16-LABEL: rcp_bf16_global_load:
@@ -148,12 +148,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; GI-FAKE16-NEXT: s_wait_xcnt 0x0
; GI-FAKE16-NEXT: s_mov_b32 s4, 0x3ff
; GI-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GI-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GI-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GI-FAKE16-NEXT: s_wait_kmcnt 0x0
; GI-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
; GI-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GI-FAKE16-NEXT: v_rcp_bf16_e32 v0, v0
-; GI-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GI-FAKE16-NEXT: v_rcp_bf16_e32 v1, v0
+; GI-FAKE16-NEXT: global_store_b16 v2, v1, s[0:1]
; GI-FAKE16-NEXT: s_endpgm
;
; GFX13-SDAG-TRUE16-LABEL: rcp_bf16_global_load:
@@ -161,12 +161,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; GFX13-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX13-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0x3ff
; GFX13-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13-SDAG-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v1, s4, v0 bitop3:0x40
; GFX13-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX13-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] scale_offset
+; GFX13-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scale_offset
; GFX13-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX13-SDAG-TRUE16-NEXT: v_rcp_bf16_e32 v0.l, v0.l
-; GFX13-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX13-SDAG-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX13-SDAG-TRUE16-NEXT: s_endpgm
;
; GFX13-SDAG-FAKE16-LABEL: rcp_bf16_global_load:
@@ -174,12 +174,12 @@ define amdgpu_kernel void @rcp_bf16_global_load(ptr addrspace(1) %out, ptr addrs
; GFX13-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX13-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0x3ff
; GFX13-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13-SDAG-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
+; GFX13-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX13-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX13-SDAG-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] scale_offset
; GFX13-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX13-SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v0, v0
-; GFX13-SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX13-SDAG-FAKE16-NEXT: v_rcp_bf16_e32 v1, v0
+; GFX13-SDAG-FAKE16-NEXT: global_store_b16 v2, v1, s[0:1]
; GFX13-SDAG-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%src.ptr = getelementptr bfloat, ptr addrspace(1) %in, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
index b5212ff1b7d8ce..a8c58eef7e66c1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
@@ -211,9 +211,9 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32,
; GFX12-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null idxen
-; GFX12-NEXT: buffer_load_b128 v[0:3], v5, s[0:3], null idxen
+; GFX12-NEXT: buffer_load_b128 v[8:11], v5, s[0:3], null idxen
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_store_b128 v[0:3], v6, s[0:3], null idxen
+; GFX12-NEXT: buffer_store_b128 v[8:11], v6, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
main_body:
call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
index 30d966eb1b7c2f..2238c00afdf129 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
@@ -165,8 +165,8 @@ define amdgpu_kernel void @sqrt_v2bf16(ptr addrspace(1) %r, ptr addrspace(1) %a)
; GFX12-FAKE16-SDAG-NEXT: s_mov_b32 s0, 0xffff
; GFX12-FAKE16-SDAG-NEXT: v_nop
; GFX12-FAKE16-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-SDAG-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_bitop2_b32 v1, s0, v1 bitop3:0x40
-; GFX12-FAKE16-SDAG-NEXT: v_sqrt_bf16_e32 v0, v0
+; GFX12-FAKE16-SDAG-NEXT: v_dual_lshrrev_b32 v2, 16, v0 :: v_dual_bitop2_b32 v1, s0, v1 bitop3:0x40
+; GFX12-FAKE16-SDAG-NEXT: v_sqrt_bf16_e32 v0, v2
; GFX12-FAKE16-SDAG-NEXT: v_nop
; GFX12-FAKE16-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
; GFX12-FAKE16-SDAG-NEXT: v_lshl_or_b32 v0, v0, 16, v1
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 34910b6f180e7c..3acaff0ed49dfd 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -2968,35 +2968,33 @@ define amdgpu_kernel void @constant_zextload_v32i1_to_v32i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s33, s2, 0x10018
; GFX1250-NEXT: s_bfe_u32 s34, s2, 0x1001c
; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1001e
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s34
+; GFX1250-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v0, s34
; GFX1250-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v2, s2
; GFX1250-NEXT: v_dual_mov_b32 v3, s16 :: v_dual_mov_b32 v4, s33
-; GFX1250-NEXT: v_dual_mov_b32 v5, s15 :: v_dual_mov_b32 v6, s31
; GFX1250-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v8, s30
; GFX1250-NEXT: v_dual_mov_b32 v9, s29 :: v_dual_mov_b32 v10, s28
-; GFX1250-NEXT: v_mov_b32_e32 v11, s13
+; GFX1250-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v12, s27
+; GFX1250-NEXT: v_dual_mov_b32 v5, s15 :: v_dual_mov_b32 v6, s31
+; GFX1250-NEXT: v_dual_mov_b32 v13, s12 :: v_dual_mov_b32 v14, s26
+; GFX1250-NEXT: v_dual_mov_b32 v15, s11 :: v_dual_mov_b32 v16, s25
+; GFX1250-NEXT: v_dual_mov_b32 v17, s10 :: v_dual_mov_b32 v18, s24
+; GFX1250-NEXT: v_dual_mov_b32 v19, s9 :: v_dual_mov_b32 v20, s23
+; GFX1250-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v22, s22
+; GFX1250-NEXT: v_dual_mov_b32 v23, s7 :: v_dual_mov_b32 v24, s21
+; GFX1250-NEXT: v_dual_mov_b32 v25, s6 :: v_dual_mov_b32 v26, s20
+; GFX1250-NEXT: v_dual_mov_b32 v27, s5 :: v_dual_mov_b32 v28, s18
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v0, s27 :: v_dual_mov_b32 v1, s12
-; GFX1250-NEXT: v_dual_mov_b32 v2, s26 :: v_dual_mov_b32 v3, s11
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v4, s25 :: v_dual_mov_b32 v5, s10
-; GFX1250-NEXT: v_dual_mov_b32 v6, s24 :: v_dual_mov_b32 v7, s9
-; GFX1250-NEXT: v_dual_mov_b32 v12, s23 :: v_dual_mov_b32 v13, s8
-; GFX1250-NEXT: v_dual_mov_b32 v14, s22 :: v_dual_mov_b32 v15, s7
-; GFX1250-NEXT: v_dual_mov_b32 v16, s21 :: v_dual_mov_b32 v17, s6
-; GFX1250-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v19, s5
-; GFX1250-NEXT: v_dual_mov_b32 v20, s18 :: v_dual_mov_b32 v21, s4
-; GFX1250-NEXT: v_dual_mov_b32 v22, s19 :: v_dual_mov_b32 v23, s3
+; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v29, s4 :: v_dual_mov_b32 v30, s19
+; GFX1250-NEXT: v_mov_b32_e32 v31, s3
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <32 x i1>, ptr addrspace(4) %in
%ext = zext <32 x i1> %load to <32 x i32>
@@ -3429,35 +3427,33 @@ define amdgpu_kernel void @constant_sextload_v32i1_to_v32i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i32 s33, s2, 0x1001e
; GFX1250-NEXT: s_bfe_i32 s34, s2, 0x1001c
; GFX1250-NEXT: s_bfe_i32 s2, s2, 0x1001d
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s34
+; GFX1250-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v0, s34
; GFX1250-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s33
; GFX1250-NEXT: v_dual_mov_b32 v3, s31 :: v_dual_mov_b32 v4, s30
-; GFX1250-NEXT: v_dual_mov_b32 v5, s29 :: v_dual_mov_b32 v6, s28
; GFX1250-NEXT: v_dual_mov_b32 v7, s27 :: v_dual_mov_b32 v8, s26
; GFX1250-NEXT: v_dual_mov_b32 v9, s25 :: v_dual_mov_b32 v10, s24
-; GFX1250-NEXT: v_mov_b32_e32 v11, s23
+; GFX1250-NEXT: v_dual_mov_b32 v11, s23 :: v_dual_mov_b32 v12, s22
+; GFX1250-NEXT: v_dual_mov_b32 v5, s29 :: v_dual_mov_b32 v6, s28
+; GFX1250-NEXT: v_dual_mov_b32 v13, s21 :: v_dual_mov_b32 v14, s20
+; GFX1250-NEXT: v_dual_mov_b32 v15, s19 :: v_dual_mov_b32 v16, s18
+; GFX1250-NEXT: v_dual_mov_b32 v17, s17 :: v_dual_mov_b32 v18, s16
+; GFX1250-NEXT: v_dual_mov_b32 v19, s15 :: v_dual_mov_b32 v20, s14
+; GFX1250-NEXT: v_dual_mov_b32 v21, s13 :: v_dual_mov_b32 v22, s12
+; GFX1250-NEXT: v_dual_mov_b32 v23, s11 :: v_dual_mov_b32 v24, s10
+; GFX1250-NEXT: v_dual_mov_b32 v25, s9 :: v_dual_mov_b32 v26, s8
+; GFX1250-NEXT: v_dual_mov_b32 v27, s7 :: v_dual_mov_b32 v28, s6
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v0, s22 :: v_dual_mov_b32 v1, s21
-; GFX1250-NEXT: v_dual_mov_b32 v2, s20 :: v_dual_mov_b32 v3, s19
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v5, s17
-; GFX1250-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v7, s15
-; GFX1250-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v13, s13
-; GFX1250-NEXT: v_dual_mov_b32 v14, s12 :: v_dual_mov_b32 v15, s11
-; GFX1250-NEXT: v_dual_mov_b32 v16, s10 :: v_dual_mov_b32 v17, s9
-; GFX1250-NEXT: v_dual_mov_b32 v18, s8 :: v_dual_mov_b32 v19, s7
-; GFX1250-NEXT: v_dual_mov_b32 v20, s6 :: v_dual_mov_b32 v21, s5
-; GFX1250-NEXT: v_dual_mov_b32 v22, s4 :: v_dual_mov_b32 v23, s3
+; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v29, s5 :: v_dual_mov_b32 v30, s4
+; GFX1250-NEXT: v_mov_b32_e32 v31, s3
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <32 x i1>, ptr addrspace(4) %in
%ext = sext <32 x i1> %load to <32 x i32>
@@ -4147,21 +4143,9 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s31, s3, 0x10019
; GFX1250-NEXT: s_bfe_u32 s63, s3, 0x1001a
; GFX1250-NEXT: s_bfe_u32 s64, s3, 0x10018
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s65
-; GFX1250-NEXT: s_bfe_u32 s29, s3, 0x10017
-; GFX1250-NEXT: s_bfe_u32 s60, s3, 0x10016
-; GFX1250-NEXT: s_bfe_u32 s61, s3, 0x10015
-; GFX1250-NEXT: s_bfe_u32 s62, s3, 0x10014
-; GFX1250-NEXT: v_dual_mov_b32 v1, s34 :: v_dual_mov_b32 v2, s66
-; GFX1250-NEXT: v_dual_mov_b32 v3, s33 :: v_dual_mov_b32 v4, s64
-; GFX1250-NEXT: s_bfe_u32 s27, s3, 0x10013
-; GFX1250-NEXT: s_bfe_u32 s28, s3, 0x10011
-; GFX1250-NEXT: s_bfe_u32 s58, s3, 0x10012
-; GFX1250-NEXT: s_bfe_u32 s59, s3, 0x10010
-; GFX1250-NEXT: v_dual_mov_b32 v5, s31 :: v_dual_mov_b32 v6, s63
-; GFX1250-NEXT: v_dual_mov_b32 v7, s30 :: v_dual_mov_b32 v8, s62
-; GFX1250-NEXT: v_dual_mov_b32 v9, s61 :: v_dual_mov_b32 v10, s60
-; GFX1250-NEXT: v_dual_mov_b32 v11, s29 :: v_dual_mov_b32 v12, s59
+; GFX1250-NEXT: v_dual_mov_b32 v52, 0 :: v_dual_mov_b32 v0, s65
+; GFX1250-NEXT: s_bfe_u32 s14, s2, 0x10017
+; GFX1250-NEXT: s_bfe_u32 s15, s2, 0x1001b
; GFX1250-NEXT: s_bfe_u32 s19, s3, 0x10003
; GFX1250-NEXT: s_bfe_u32 s20, s3, 0x10001
; GFX1250-NEXT: s_bfe_u32 s21, s3, 0x10007
@@ -4170,6 +4154,12 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s24, s3, 0x10009
; GFX1250-NEXT: s_bfe_u32 s25, s3, 0x1000f
; GFX1250-NEXT: s_bfe_u32 s26, s3, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s27, s3, 0x10013
+; GFX1250-NEXT: s_bfe_u32 s28, s3, 0x10011
+; GFX1250-NEXT: s_bfe_u32 s29, s3, 0x10017
+; GFX1250-NEXT: s_bfe_u32 s45, s2, 0x10016
+; GFX1250-NEXT: s_bfe_u32 s46, s2, 0x10015
+; GFX1250-NEXT: s_bfe_u32 s47, s2, 0x10014
; GFX1250-NEXT: s_and_b32 s51, s3, 1
; GFX1250-NEXT: s_bfe_u32 s52, s3, 0x10002
; GFX1250-NEXT: s_bfe_u32 s53, s3, 0x10006
@@ -4177,34 +4167,45 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s55, s3, 0x1000a
; GFX1250-NEXT: s_bfe_u32 s56, s3, 0x10008
; GFX1250-NEXT: s_bfe_u32 s57, s3, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v13, s28 :: v_dual_mov_b32 v14, s58
+; GFX1250-NEXT: s_bfe_u32 s58, s3, 0x10012
+; GFX1250-NEXT: s_bfe_u32 s59, s3, 0x10010
+; GFX1250-NEXT: s_bfe_u32 s60, s3, 0x10016
+; GFX1250-NEXT: s_bfe_u32 s61, s3, 0x10015
+; GFX1250-NEXT: s_bfe_u32 s62, s3, 0x10014
+; GFX1250-NEXT: v_dual_mov_b32 v1, s34 :: v_dual_mov_b32 v2, s66
+; GFX1250-NEXT: v_dual_mov_b32 v3, s33 :: v_dual_mov_b32 v4, s64
; GFX1250-NEXT: s_bfe_u32 s3, s3, 0x1000c
-; GFX1250-NEXT: v_mov_b32_e32 v15, s27
-; GFX1250-NEXT: s_clause 0x3
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:240
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:224
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:208
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:192
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s26
-; GFX1250-NEXT: v_dual_mov_b32 v2, s57 :: v_dual_mov_b32 v3, s25
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v4, s56 :: v_dual_mov_b32 v5, s24
-; GFX1250-NEXT: v_dual_mov_b32 v6, s55 :: v_dual_mov_b32 v7, s23
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_mov_b32_e32 v8, s54
+; GFX1250-NEXT: s_bfe_u32 s12, s2, 0x10013
+; GFX1250-NEXT: s_bfe_u32 s13, s2, 0x10011
+; GFX1250-NEXT: s_bfe_u32 s43, s2, 0x10012
+; GFX1250-NEXT: s_bfe_u32 s44, s2, 0x10010
+; GFX1250-NEXT: v_dual_mov_b32 v5, s31 :: v_dual_mov_b32 v6, s63
+; GFX1250-NEXT: v_dual_mov_b32 v7, s30 :: v_dual_mov_b32 v8, s62
+; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000f
+; GFX1250-NEXT: s_bfe_u32 s11, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s41, s2, 0x1000e
+; GFX1250-NEXT: s_bfe_u32 s42, s2, 0x1000c
+; GFX1250-NEXT: v_dual_mov_b32 v9, s61 :: v_dual_mov_b32 v10, s60
+; GFX1250-NEXT: v_dual_mov_b32 v11, s29 :: v_dual_mov_b32 v12, s59
+; GFX1250-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v16, s3
+; GFX1250-NEXT: v_dual_mov_b32 v17, s26 :: v_dual_mov_b32 v18, s57
+; GFX1250-NEXT: v_dual_mov_b32 v19, s25 :: v_dual_mov_b32 v20, s56
+; GFX1250-NEXT: v_dual_mov_b32 v39, s15 :: v_dual_mov_b32 v40, s47
+; GFX1250-NEXT: v_dual_mov_b32 v41, s46 :: v_dual_mov_b32 v42, s45
+; GFX1250-NEXT: v_dual_mov_b32 v43, s14 :: v_dual_mov_b32 v44, s44
+; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s9, s2, 0x10009
+; GFX1250-NEXT: s_bfe_u32 s39, s2, 0x1000a
+; GFX1250-NEXT: s_bfe_u32 s40, s2, 0x10008
+; GFX1250-NEXT: v_dual_mov_b32 v13, s28 :: v_dual_mov_b32 v14, s58
+; GFX1250-NEXT: v_dual_mov_b32 v21, s24 :: v_dual_mov_b32 v22, s55
+; GFX1250-NEXT: v_dual_mov_b32 v23, s23 :: v_dual_mov_b32 v24, s54
+; GFX1250-NEXT: v_dual_mov_b32 v45, s13 :: v_dual_mov_b32 v46, s43
+; GFX1250-NEXT: v_dual_mov_b32 v47, s12 :: v_dual_mov_b32 v48, s42
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10001
; GFX1250-NEXT: s_bfe_u32 s6, s2, 0x10007
; GFX1250-NEXT: s_bfe_u32 s7, s2, 0x10005
-; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x1000b
-; GFX1250-NEXT: s_bfe_u32 s9, s2, 0x10009
-; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000f
-; GFX1250-NEXT: s_bfe_u32 s11, s2, 0x1000d
-; GFX1250-NEXT: s_bfe_u32 s12, s2, 0x10013
-; GFX1250-NEXT: s_bfe_u32 s13, s2, 0x10011
-; GFX1250-NEXT: s_bfe_u32 s14, s2, 0x10017
-; GFX1250-NEXT: s_bfe_u32 s15, s2, 0x1001b
; GFX1250-NEXT: s_bfe_u32 s16, s2, 0x10019
; GFX1250-NEXT: s_lshr_b32 s17, s2, 31
; GFX1250-NEXT: s_bfe_u32 s18, s2, 0x1001d
@@ -4212,60 +4213,48 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s36, s2, 0x10002
; GFX1250-NEXT: s_bfe_u32 s37, s2, 0x10006
; GFX1250-NEXT: s_bfe_u32 s38, s2, 0x10004
-; GFX1250-NEXT: s_bfe_u32 s39, s2, 0x1000a
-; GFX1250-NEXT: s_bfe_u32 s40, s2, 0x10008
-; GFX1250-NEXT: s_bfe_u32 s41, s2, 0x1000e
-; GFX1250-NEXT: s_bfe_u32 s42, s2, 0x1000c
-; GFX1250-NEXT: s_bfe_u32 s43, s2, 0x10012
-; GFX1250-NEXT: s_bfe_u32 s44, s2, 0x10010
-; GFX1250-NEXT: s_bfe_u32 s45, s2, 0x10016
-; GFX1250-NEXT: s_bfe_u32 s46, s2, 0x10015
-; GFX1250-NEXT: s_bfe_u32 s47, s2, 0x10014
; GFX1250-NEXT: s_bfe_u32 s48, s2, 0x1001a
; GFX1250-NEXT: s_bfe_u32 s49, s2, 0x10018
; GFX1250-NEXT: s_bfe_u32 s50, s2, 0x1001e
; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1001c
-; GFX1250-NEXT: v_dual_mov_b32 v9, s22 :: v_dual_mov_b32 v10, s53
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v11, s21 :: v_dual_mov_b32 v12, s51
-; GFX1250-NEXT: v_dual_mov_b32 v13, s20 :: v_dual_mov_b32 v14, s52
-; GFX1250-NEXT: v_dual_mov_b32 v15, s19 :: v_dual_mov_b32 v16, s2
-; GFX1250-NEXT: v_dual_mov_b32 v17, s18 :: v_dual_mov_b32 v18, s50
-; GFX1250-NEXT: v_dual_mov_b32 v19, s17 :: v_dual_mov_b32 v20, s49
-; GFX1250-NEXT: v_dual_mov_b32 v21, s16 :: v_dual_mov_b32 v22, s48
-; GFX1250-NEXT: v_mov_b32_e32 v23, s15
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:176
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:160
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:144
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:128
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x5
-; GFX1250-NEXT: v_dual_mov_b32 v0, s47 :: v_dual_mov_b32 v1, s46
-; GFX1250-NEXT: v_dual_mov_b32 v2, s45 :: v_dual_mov_b32 v3, s14
-; GFX1250-NEXT: s_wait_xcnt 0x4
-; GFX1250-NEXT: v_dual_mov_b32 v4, s44 :: v_dual_mov_b32 v5, s13
-; GFX1250-NEXT: v_dual_mov_b32 v6, s43 :: v_dual_mov_b32 v7, s12
+; GFX1250-NEXT: v_dual_mov_b32 v25, s22 :: v_dual_mov_b32 v26, s53
+; GFX1250-NEXT: v_dual_mov_b32 v27, s21 :: v_dual_mov_b32 v28, s51
+; GFX1250-NEXT: v_dual_mov_b32 v49, s11 :: v_dual_mov_b32 v50, s41
+; GFX1250-NEXT: v_mov_b32_e32 v51, s10
+; GFX1250-NEXT: s_clause 0x3
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:240
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:224
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1] offset:208
+; GFX1250-NEXT: global_store_b128 v52, v[12:15], s[0:1] offset:192
+; GFX1250-NEXT: v_dual_mov_b32 v29, s20 :: v_dual_mov_b32 v30, s52
+; GFX1250-NEXT: v_dual_mov_b32 v31, s19 :: v_dual_mov_b32 v32, s2
; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v8, s42 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s41 :: v_dual_mov_b32 v11, s10
+; GFX1250-NEXT: v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v1, s9
+; GFX1250-NEXT: v_dual_mov_b32 v2, s39 :: v_dual_mov_b32 v3, s8
; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v12, s40 :: v_dual_mov_b32 v13, s9
-; GFX1250-NEXT: v_dual_mov_b32 v14, s39 :: v_dual_mov_b32 v15, s8
+; GFX1250-NEXT: v_dual_mov_b32 v4, s38 :: v_dual_mov_b32 v33, s18
+; GFX1250-NEXT: v_dual_mov_b32 v34, s50 :: v_dual_mov_b32 v35, s17
+; GFX1250-NEXT: v_dual_mov_b32 v36, s49 :: v_dual_mov_b32 v5, s7
+; GFX1250-NEXT: v_dual_mov_b32 v6, s37 :: v_dual_mov_b32 v7, s6
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v16, s38 :: v_dual_mov_b32 v17, s7
-; GFX1250-NEXT: v_dual_mov_b32 v18, s37 :: v_dual_mov_b32 v19, s6
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v20, s35 :: v_dual_mov_b32 v21, s5
-; GFX1250-NEXT: v_dual_mov_b32 v22, s36 :: v_dual_mov_b32 v23, s4
+; GFX1250-NEXT: v_dual_mov_b32 v8, s35 :: v_dual_mov_b32 v37, s16
+; GFX1250-NEXT: v_mov_b32_e32 v38, s48
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v52, v[16:19], s[0:1] offset:176
+; GFX1250-NEXT: global_store_b128 v52, v[20:23], s[0:1] offset:160
+; GFX1250-NEXT: global_store_b128 v52, v[24:27], s[0:1] offset:144
+; GFX1250-NEXT: global_store_b128 v52, v[28:31], s[0:1] offset:128
+; GFX1250-NEXT: global_store_b128 v52, v[32:35], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v52, v[36:39], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v10, s36
+; GFX1250-NEXT: v_mov_b32_e32 v11, s4
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX1250-NEXT: global_store_b128 v52, v[40:43], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v52, v[44:47], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v52, v[48:51], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <64 x i1>, ptr addrspace(4) %in
%ext = zext <64 x i1> %load to <64 x i32>
@@ -5021,21 +5010,12 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i32 s60, s3, 0x1001a
; GFX1250-NEXT: s_bfe_i32 s61, s3, 0x10019
; GFX1250-NEXT: s_bfe_i32 s62, s3, 0x10018
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s65
-; GFX1250-NEXT: s_bfe_i32 s55, s3, 0x10017
-; GFX1250-NEXT: s_bfe_i32 s56, s3, 0x10016
-; GFX1250-NEXT: s_bfe_i32 s57, s3, 0x10015
-; GFX1250-NEXT: s_bfe_i32 s58, s3, 0x10014
-; GFX1250-NEXT: v_dual_mov_b32 v1, s66 :: v_dual_mov_b32 v2, s64
-; GFX1250-NEXT: v_dual_mov_b32 v3, s63 :: v_dual_mov_b32 v4, s62
-; GFX1250-NEXT: s_bfe_i32 s51, s3, 0x10013
-; GFX1250-NEXT: s_bfe_i32 s52, s3, 0x10012
-; GFX1250-NEXT: s_bfe_i32 s53, s3, 0x10011
-; GFX1250-NEXT: s_bfe_i32 s54, s3, 0x10010
-; GFX1250-NEXT: v_dual_mov_b32 v5, s61 :: v_dual_mov_b32 v6, s60
-; GFX1250-NEXT: v_dual_mov_b32 v7, s59 :: v_dual_mov_b32 v8, s58
-; GFX1250-NEXT: v_dual_mov_b32 v9, s57 :: v_dual_mov_b32 v10, s56
-; GFX1250-NEXT: v_dual_mov_b32 v11, s55 :: v_dual_mov_b32 v12, s54
+; GFX1250-NEXT: v_dual_mov_b32 v52, 0 :: v_dual_mov_b32 v0, s65
+; GFX1250-NEXT: s_bfe_i32 s24, s2, 0x10017
+; GFX1250-NEXT: s_bfe_i32 s25, s2, 0x10016
+; GFX1250-NEXT: s_bfe_i32 s26, s2, 0x10015
+; GFX1250-NEXT: s_bfe_i32 s27, s2, 0x10014
+; GFX1250-NEXT: s_bfe_i32 s28, s2, 0x1001b
; GFX1250-NEXT: s_bfe_i32 s36, s3, 0x10003
; GFX1250-NEXT: s_bfe_i32 s37, s3, 0x10002
; GFX1250-NEXT: s_bfe_i32 s38, s3, 0x10001
@@ -5051,22 +5031,44 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i32 s48, s3, 0x1000f
; GFX1250-NEXT: s_bfe_i32 s49, s3, 0x1000e
; GFX1250-NEXT: s_bfe_i32 s50, s3, 0x1000d
-; GFX1250-NEXT: v_dual_mov_b32 v13, s53 :: v_dual_mov_b32 v14, s52
+; GFX1250-NEXT: s_bfe_i32 s51, s3, 0x10013
+; GFX1250-NEXT: s_bfe_i32 s52, s3, 0x10012
+; GFX1250-NEXT: s_bfe_i32 s53, s3, 0x10011
+; GFX1250-NEXT: s_bfe_i32 s54, s3, 0x10010
+; GFX1250-NEXT: s_bfe_i32 s55, s3, 0x10017
+; GFX1250-NEXT: s_bfe_i32 s56, s3, 0x10016
+; GFX1250-NEXT: s_bfe_i32 s57, s3, 0x10015
+; GFX1250-NEXT: s_bfe_i32 s58, s3, 0x10014
+; GFX1250-NEXT: v_dual_mov_b32 v1, s66 :: v_dual_mov_b32 v2, s64
+; GFX1250-NEXT: v_dual_mov_b32 v3, s63 :: v_dual_mov_b32 v4, s62
; GFX1250-NEXT: s_bfe_i32 s3, s3, 0x1000c
-; GFX1250-NEXT: v_mov_b32_e32 v15, s51
-; GFX1250-NEXT: s_clause 0x3
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:240
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:224
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:208
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:192
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s50
-; GFX1250-NEXT: v_dual_mov_b32 v2, s49 :: v_dual_mov_b32 v3, s48
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v4, s47 :: v_dual_mov_b32 v5, s46
-; GFX1250-NEXT: v_dual_mov_b32 v6, s45 :: v_dual_mov_b32 v7, s44
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_mov_b32_e32 v8, s43
+; GFX1250-NEXT: s_bfe_i32 s20, s2, 0x10013
+; GFX1250-NEXT: s_bfe_i32 s21, s2, 0x10012
+; GFX1250-NEXT: s_bfe_i32 s22, s2, 0x10011
+; GFX1250-NEXT: s_bfe_i32 s23, s2, 0x10010
+; GFX1250-NEXT: v_dual_mov_b32 v5, s61 :: v_dual_mov_b32 v6, s60
+; GFX1250-NEXT: v_dual_mov_b32 v7, s59 :: v_dual_mov_b32 v8, s58
+; GFX1250-NEXT: s_bfe_i32 s16, s2, 0x1000f
+; GFX1250-NEXT: s_bfe_i32 s17, s2, 0x1000e
+; GFX1250-NEXT: s_bfe_i32 s18, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_i32 s19, s2, 0x1000c
+; GFX1250-NEXT: v_dual_mov_b32 v9, s57 :: v_dual_mov_b32 v10, s56
+; GFX1250-NEXT: v_dual_mov_b32 v11, s55 :: v_dual_mov_b32 v12, s54
+; GFX1250-NEXT: v_dual_mov_b32 v15, s51 :: v_dual_mov_b32 v16, s3
+; GFX1250-NEXT: v_dual_mov_b32 v17, s50 :: v_dual_mov_b32 v18, s49
+; GFX1250-NEXT: v_dual_mov_b32 v19, s48 :: v_dual_mov_b32 v20, s47
+; GFX1250-NEXT: v_dual_mov_b32 v39, s28 :: v_dual_mov_b32 v40, s27
+; GFX1250-NEXT: v_dual_mov_b32 v41, s26 :: v_dual_mov_b32 v42, s25
+; GFX1250-NEXT: v_dual_mov_b32 v43, s24 :: v_dual_mov_b32 v44, s23
+; GFX1250-NEXT: s_bfe_i32 s12, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_i32 s13, s2, 0x1000a
+; GFX1250-NEXT: s_bfe_i32 s14, s2, 0x10009
+; GFX1250-NEXT: s_bfe_i32 s15, s2, 0x10008
+; GFX1250-NEXT: v_dual_mov_b32 v13, s53 :: v_dual_mov_b32 v14, s52
+; GFX1250-NEXT: v_dual_mov_b32 v21, s46 :: v_dual_mov_b32 v22, s45
+; GFX1250-NEXT: v_dual_mov_b32 v23, s44 :: v_dual_mov_b32 v24, s43
+; GFX1250-NEXT: v_dual_mov_b32 v45, s22 :: v_dual_mov_b32 v46, s21
+; GFX1250-NEXT: v_dual_mov_b32 v47, s20 :: v_dual_mov_b32 v48, s19
; GFX1250-NEXT: s_bfe_i32 s4, s2, 0x10003
; GFX1250-NEXT: s_bfe_i32 s5, s2, 0x10002
; GFX1250-NEXT: s_bfe_i32 s6, s2, 0x10001
@@ -5075,23 +5077,6 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i32 s9, s2, 0x10006
; GFX1250-NEXT: s_bfe_i32 s10, s2, 0x10005
; GFX1250-NEXT: s_bfe_i32 s11, s2, 0x10004
-; GFX1250-NEXT: s_bfe_i32 s12, s2, 0x1000b
-; GFX1250-NEXT: s_bfe_i32 s13, s2, 0x1000a
-; GFX1250-NEXT: s_bfe_i32 s14, s2, 0x10009
-; GFX1250-NEXT: s_bfe_i32 s15, s2, 0x10008
-; GFX1250-NEXT: s_bfe_i32 s16, s2, 0x1000f
-; GFX1250-NEXT: s_bfe_i32 s17, s2, 0x1000e
-; GFX1250-NEXT: s_bfe_i32 s18, s2, 0x1000d
-; GFX1250-NEXT: s_bfe_i32 s19, s2, 0x1000c
-; GFX1250-NEXT: s_bfe_i32 s20, s2, 0x10013
-; GFX1250-NEXT: s_bfe_i32 s21, s2, 0x10012
-; GFX1250-NEXT: s_bfe_i32 s22, s2, 0x10011
-; GFX1250-NEXT: s_bfe_i32 s23, s2, 0x10010
-; GFX1250-NEXT: s_bfe_i32 s24, s2, 0x10017
-; GFX1250-NEXT: s_bfe_i32 s25, s2, 0x10016
-; GFX1250-NEXT: s_bfe_i32 s26, s2, 0x10015
-; GFX1250-NEXT: s_bfe_i32 s27, s2, 0x10014
-; GFX1250-NEXT: s_bfe_i32 s28, s2, 0x1001b
; GFX1250-NEXT: s_bfe_i32 s29, s2, 0x1001a
; GFX1250-NEXT: s_bfe_i32 s30, s2, 0x10019
; GFX1250-NEXT: s_bfe_i32 s31, s2, 0x10018
@@ -5099,47 +5084,44 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i32 s34, s2, 0x1001e
; GFX1250-NEXT: s_bfe_i32 s35, s2, 0x1001d
; GFX1250-NEXT: s_bfe_i32 s2, s2, 0x1001c
-; GFX1250-NEXT: v_dual_mov_b32 v9, s42 :: v_dual_mov_b32 v10, s41
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v11, s40 :: v_dual_mov_b32 v12, s39
-; GFX1250-NEXT: v_dual_mov_b32 v13, s38 :: v_dual_mov_b32 v14, s37
-; GFX1250-NEXT: v_dual_mov_b32 v15, s36 :: v_dual_mov_b32 v16, s2
-; GFX1250-NEXT: v_dual_mov_b32 v17, s35 :: v_dual_mov_b32 v18, s34
-; GFX1250-NEXT: v_dual_mov_b32 v19, s33 :: v_dual_mov_b32 v20, s31
-; GFX1250-NEXT: v_dual_mov_b32 v21, s30 :: v_dual_mov_b32 v22, s29
-; GFX1250-NEXT: v_mov_b32_e32 v23, s28
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:176
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:160
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:144
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:128
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x5
-; GFX1250-NEXT: v_dual_mov_b32 v0, s27 :: v_dual_mov_b32 v1, s26
-; GFX1250-NEXT: v_dual_mov_b32 v2, s25 :: v_dual_mov_b32 v3, s24
-; GFX1250-NEXT: s_wait_xcnt 0x4
-; GFX1250-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v5, s22
-; GFX1250-NEXT: v_dual_mov_b32 v6, s21 :: v_dual_mov_b32 v7, s20
+; GFX1250-NEXT: v_dual_mov_b32 v25, s42 :: v_dual_mov_b32 v26, s41
+; GFX1250-NEXT: v_dual_mov_b32 v27, s40 :: v_dual_mov_b32 v28, s39
+; GFX1250-NEXT: v_dual_mov_b32 v49, s18 :: v_dual_mov_b32 v50, s17
+; GFX1250-NEXT: v_mov_b32_e32 v51, s16
+; GFX1250-NEXT: s_clause 0x3
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:240
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:224
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1] offset:208
+; GFX1250-NEXT: global_store_b128 v52, v[12:15], s[0:1] offset:192
+; GFX1250-NEXT: v_dual_mov_b32 v29, s38 :: v_dual_mov_b32 v30, s37
+; GFX1250-NEXT: v_dual_mov_b32 v31, s36 :: v_dual_mov_b32 v32, s2
; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v8, s19 :: v_dual_mov_b32 v9, s18
-; GFX1250-NEXT: v_dual_mov_b32 v10, s17 :: v_dual_mov_b32 v11, s16
+; GFX1250-NEXT: v_dual_mov_b32 v0, s15 :: v_dual_mov_b32 v1, s14
+; GFX1250-NEXT: v_dual_mov_b32 v2, s13 :: v_dual_mov_b32 v3, s12
; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v12, s15 :: v_dual_mov_b32 v13, s14
-; GFX1250-NEXT: v_dual_mov_b32 v14, s13 :: v_dual_mov_b32 v15, s12
+; GFX1250-NEXT: v_dual_mov_b32 v4, s11 :: v_dual_mov_b32 v33, s35
+; GFX1250-NEXT: v_dual_mov_b32 v34, s34 :: v_dual_mov_b32 v35, s33
+; GFX1250-NEXT: v_dual_mov_b32 v36, s31 :: v_dual_mov_b32 v5, s10
+; GFX1250-NEXT: v_dual_mov_b32 v6, s9 :: v_dual_mov_b32 v7, s8
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v16, s11 :: v_dual_mov_b32 v17, s10
-; GFX1250-NEXT: v_dual_mov_b32 v18, s9 :: v_dual_mov_b32 v19, s8
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
-; GFX1250-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
+; GFX1250-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v37, s30
+; GFX1250-NEXT: v_mov_b32_e32 v38, s29
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v52, v[16:19], s[0:1] offset:176
+; GFX1250-NEXT: global_store_b128 v52, v[20:23], s[0:1] offset:160
+; GFX1250-NEXT: global_store_b128 v52, v[24:27], s[0:1] offset:144
+; GFX1250-NEXT: global_store_b128 v52, v[28:31], s[0:1] offset:128
+; GFX1250-NEXT: global_store_b128 v52, v[32:35], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v52, v[36:39], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v9, s6 :: v_dual_mov_b32 v10, s5
+; GFX1250-NEXT: v_mov_b32_e32 v11, s4
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX1250-NEXT: global_store_b128 v52, v[40:43], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v52, v[44:47], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v52, v[48:51], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <64 x i1>, ptr addrspace(4) %in
%ext = sext <64 x i1> %load to <64 x i32>
@@ -9337,127 +9319,119 @@ define amdgpu_kernel void @constant_sextload_v32i1_to_v32i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_lshr_b32 s40, s2, 29
; GFX1250-NEXT: s_lshr_b32 s42, s2, 26
; GFX1250-NEXT: s_lshr_b32 s44, s2, 27
-; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s46, s2, 24
; GFX1250-NEXT: s_lshr_b32 s48, s2, 25
-; GFX1250-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s34
-; GFX1250-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v1, s35 :: v_dual_mov_b32 v2, s36
-; GFX1250-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, s38
; GFX1250-NEXT: s_lshr_b32 s26, s2, 22
; GFX1250-NEXT: s_lshr_b32 s50, s2, 23
-; GFX1250-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s39 :: v_dual_mov_b32 v6, s40
-; GFX1250-NEXT: v_dual_mov_b32 v7, s41 :: v_dual_mov_b32 v8, s42
-; GFX1250-NEXT: s_lshr_b32 s52, s2, 20
-; GFX1250-NEXT: s_lshr_b32 s54, s2, 21
-; GFX1250-NEXT: v_dual_mov_b32 v9, s43 :: v_dual_mov_b32 v10, s44
-; GFX1250-NEXT: v_dual_mov_b32 v11, s45 :: v_dual_mov_b32 v12, s46
-; GFX1250-NEXT: s_lshr_b32 s56, s2, 18
-; GFX1250-NEXT: s_lshr_b32 s58, s2, 19
-; GFX1250-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v13, s47 :: v_dual_mov_b32 v14, s48
-; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX1250-NEXT: v_mov_b32_e32 v15, s49
-; GFX1250-NEXT: s_lshr_b32 s60, s2, 16
-; GFX1250-NEXT: s_lshr_b32 s62, s2, 17
-; GFX1250-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s64, s2, 14
-; GFX1250-NEXT: s_lshr_b32 s66, s2, 15
-; GFX1250-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x10000
-; GFX1250-NEXT: s_clause 0x3
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:240
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:224
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:208
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:192
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v0, s26 :: v_dual_mov_b32 v1, s27
-; GFX1250-NEXT: v_dual_mov_b32 v2, s50 :: v_dual_mov_b32 v3, s51
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_mov_b32_e32 v4, s52
-; GFX1250-NEXT: s_lshr_b32 s30, s2, 12
; GFX1250-NEXT: s_lshr_b32 s28, s2, 13
; GFX1250-NEXT: s_lshr_b32 s24, s2, 10
; GFX1250-NEXT: s_lshr_b32 s22, s2, 11
-; GFX1250-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s53 :: v_dual_mov_b32 v6, s54
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v7, s55 :: v_dual_mov_b32 v8, s56
+; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s46, s2, 24
+; GFX1250-NEXT: s_lshr_b32 s52, s2, 20
+; GFX1250-NEXT: s_lshr_b32 s54, s2, 21
; GFX1250-NEXT: s_lshr_b32 s20, s2, 8
; GFX1250-NEXT: s_lshr_b32 s18, s2, 9
-; GFX1250-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v9, s57 :: v_dual_mov_b32 v10, s58
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v11, s59 :: v_dual_mov_b32 v12, s60
+; GFX1250-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v52, 0 :: v_dual_mov_b32 v0, s34
+; GFX1250-NEXT: s_lshr_b32 s56, s2, 18
+; GFX1250-NEXT: s_lshr_b32 s58, s2, 19
; GFX1250-NEXT: s_lshr_b32 s16, s2, 6
; GFX1250-NEXT: s_lshr_b32 s14, s2, 7
; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v13, s61 :: v_dual_mov_b32 v14, s62
-; GFX1250-NEXT: v_dual_mov_b32 v15, s63 :: v_dual_mov_b32 v16, s64
+; GFX1250-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v1, s35 :: v_dual_mov_b32 v2, s36
+; GFX1250-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, s38
+; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s60, s2, 16
+; GFX1250-NEXT: s_lshr_b32 s62, s2, 17
; GFX1250-NEXT: s_lshr_b32 s12, s2, 4
; GFX1250-NEXT: s_lshr_b32 s10, s2, 5
; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v17, s65 :: v_dual_mov_b32 v18, s66
-; GFX1250-NEXT: v_dual_mov_b32 v19, s67 :: v_dual_mov_b32 v20, s30
+; GFX1250-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v5, s39 :: v_dual_mov_b32 v6, s40
+; GFX1250-NEXT: v_dual_mov_b32 v7, s41 :: v_dual_mov_b32 v8, s42
+; GFX1250-NEXT: s_lshr_b32 s64, s2, 14
+; GFX1250-NEXT: s_lshr_b32 s66, s2, 15
; GFX1250-NEXT: s_lshr_b32 s8, s2, 2
; GFX1250-NEXT: s_lshr_b32 s6, s2, 3
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v21, s31 :: v_dual_mov_b32 v22, s28
-; GFX1250-NEXT: v_mov_b32_e32 v23, s29
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:176
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:160
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:144
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:128
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x5
-; GFX1250-NEXT: v_dual_mov_b32 v0, s24 :: v_dual_mov_b32 v1, s25
-; GFX1250-NEXT: v_dual_mov_b32 v2, s22 :: v_dual_mov_b32 v3, s23
-; GFX1250-NEXT: s_wait_xcnt 0x4
-; GFX1250-NEXT: v_mov_b32_e32 v4, s20
+; GFX1250-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v9, s43 :: v_dual_mov_b32 v10, s44
+; GFX1250-NEXT: v_dual_mov_b32 v11, s45 :: v_dual_mov_b32 v12, s46
+; GFX1250-NEXT: v_dual_mov_b32 v15, s49 :: v_dual_mov_b32 v16, s26
+; GFX1250-NEXT: v_dual_mov_b32 v17, s27 :: v_dual_mov_b32 v18, s50
+; GFX1250-NEXT: v_dual_mov_b32 v19, s51 :: v_dual_mov_b32 v20, s52
+; GFX1250-NEXT: v_dual_mov_b32 v39, s29 :: v_dual_mov_b32 v40, s24
+; GFX1250-NEXT: v_dual_mov_b32 v41, s25 :: v_dual_mov_b32 v42, s22
+; GFX1250-NEXT: v_dual_mov_b32 v43, s23 :: v_dual_mov_b32 v44, s20
+; GFX1250-NEXT: s_lshr_b32 s30, s2, 12
; GFX1250-NEXT: s_lshr_b32 s68, s2, 1
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v6, s18
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v7, s19 :: v_dual_mov_b32 v8, s16
+; GFX1250-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v13, s47 :: v_dual_mov_b32 v14, s48
+; GFX1250-NEXT: v_dual_mov_b32 v21, s53 :: v_dual_mov_b32 v22, s54
+; GFX1250-NEXT: v_dual_mov_b32 v23, s55 :: v_dual_mov_b32 v24, s56
+; GFX1250-NEXT: v_dual_mov_b32 v45, s21 :: v_dual_mov_b32 v46, s18
+; GFX1250-NEXT: v_dual_mov_b32 v47, s19 :: v_dual_mov_b32 v48, s16
; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v9, s17 :: v_dual_mov_b32 v10, s14
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v11, s15 :: v_dual_mov_b32 v12, s12
+; GFX1250-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[64:65], s[64:65], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v25, s57 :: v_dual_mov_b32 v26, s58
+; GFX1250-NEXT: v_dual_mov_b32 v27, s59 :: v_dual_mov_b32 v28, s60
+; GFX1250-NEXT: v_dual_mov_b32 v49, s17 :: v_dual_mov_b32 v50, s14
+; GFX1250-NEXT: v_mov_b32_e32 v51, s15
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[68:69], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v13, s13 :: v_dual_mov_b32 v14, s10
+; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX1250-NEXT: s_clause 0x3
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:240
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:224
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1] offset:208
+; GFX1250-NEXT: global_store_b128 v52, v[12:15], s[0:1] offset:192
+; GFX1250-NEXT: v_dual_mov_b32 v29, s61 :: v_dual_mov_b32 v30, s62
+; GFX1250-NEXT: v_dual_mov_b32 v31, s63 :: v_dual_mov_b32 v32, s64
+; GFX1250-NEXT: s_wait_xcnt 0x3
+; GFX1250-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
+; GFX1250-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
+; GFX1250-NEXT: s_wait_xcnt 0x2
+; GFX1250-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v33, s65
+; GFX1250-NEXT: v_dual_mov_b32 v34, s66 :: v_dual_mov_b32 v35, s67
+; GFX1250-NEXT: v_dual_mov_b32 v36, s30 :: v_dual_mov_b32 v5, s9
+; GFX1250-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v15, s11 :: v_dual_mov_b32 v16, s8
-; GFX1250-NEXT: v_dual_mov_b32 v17, s9 :: v_dual_mov_b32 v18, s6
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v19, s7 :: v_dual_mov_b32 v20, s4
-; GFX1250-NEXT: v_dual_mov_b32 v21, s5 :: v_dual_mov_b32 v22, s2
-; GFX1250-NEXT: v_mov_b32_e32 v23, s3
+; GFX1250-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v37, s31
+; GFX1250-NEXT: v_mov_b32_e32 v38, s28
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v52, v[16:19], s[0:1] offset:176
+; GFX1250-NEXT: global_store_b128 v52, v[20:23], s[0:1] offset:160
+; GFX1250-NEXT: global_store_b128 v52, v[24:27], s[0:1] offset:144
+; GFX1250-NEXT: global_store_b128 v52, v[28:31], s[0:1] offset:128
+; GFX1250-NEXT: global_store_b128 v52, v[32:35], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v52, v[36:39], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v10, s2
+; GFX1250-NEXT: v_mov_b32_e32 v11, s3
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX1250-NEXT: global_store_b128 v52, v[40:43], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v52, v[44:47], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v52, v[48:51], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <32 x i1>, ptr addrspace(4) %in
%ext = sext <32 x i1> %load to <32 x i64>
@@ -12030,134 +12004,126 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_lshr_b32 s98, s5, 30
; GFX1250-NEXT: s_lshr_b32 s90, s5, 31
-; GFX1250-NEXT: s_lshr_b32 s96, s5, 28
-; GFX1250-NEXT: s_lshr_b32 s76, s5, 29
; GFX1250-NEXT: s_lshr_b32 s72, s5, 27
; GFX1250-NEXT: s_lshr_b32 s60, s5, 24
+; GFX1250-NEXT: s_lshr_b32 s96, s5, 28
+; GFX1250-NEXT: s_lshr_b32 s76, s5, 29
; GFX1250-NEXT: s_lshr_b32 s86, s5, 26
+; GFX1250-NEXT: s_lshr_b32 s2, s5, 17
+; GFX1250-NEXT: s_lshr_b32 s6, s5, 14
+; GFX1250-NEXT: s_lshr_b32 s8, s5, 15
+; GFX1250-NEXT: s_lshr_b32 s64, s5, 25
+; GFX1250-NEXT: s_lshr_b32 s10, s5, 12
+; GFX1250-NEXT: s_lshr_b32 s12, s5, 13
; GFX1250-NEXT: s_bfe_i64 s[98:99], s[98:99], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[90:91], s[90:91], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s64, s5, 25
-; GFX1250-NEXT: s_bfe_i64 s[96:97], s[96:97], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[76:77], s[76:77], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[72:73], s[72:73], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[60:61], s[60:61], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s98
; GFX1250-NEXT: s_lshr_b32 s54, s5, 22
; GFX1250-NEXT: s_lshr_b32 s42, s5, 23
-; GFX1250-NEXT: v_dual_mov_b32 v1, s99 :: v_dual_mov_b32 v2, s90
-; GFX1250-NEXT: v_dual_mov_b32 v3, s91 :: v_dual_mov_b32 v4, s96
+; GFX1250-NEXT: s_lshr_b32 s14, s5, 10
+; GFX1250-NEXT: s_lshr_b32 s18, s5, 11
+; GFX1250-NEXT: s_bfe_i64 s[96:97], s[96:97], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[76:77], s[76:77], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[86:87], s[86:87], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s98
; GFX1250-NEXT: s_lshr_b32 s50, s5, 20
; GFX1250-NEXT: s_lshr_b32 s28, s5, 21
-; GFX1250-NEXT: s_lshr_b32 s2, s5, 17
-; GFX1250-NEXT: s_lshr_b32 s6, s5, 14
-; GFX1250-NEXT: s_lshr_b32 s8, s5, 15
-; GFX1250-NEXT: v_dual_mov_b32 v5, s97 :: v_dual_mov_b32 v6, s76
-; GFX1250-NEXT: v_dual_mov_b32 v7, s77 :: v_dual_mov_b32 v10, s86
+; GFX1250-NEXT: s_lshr_b32 s20, s5, 8
+; GFX1250-NEXT: s_lshr_b32 s22, s5, 9
+; GFX1250-NEXT: v_dual_mov_b32 v1, s99 :: v_dual_mov_b32 v2, s90
+; GFX1250-NEXT: v_dual_mov_b32 v3, s91 :: v_dual_mov_b32 v4, s96
; GFX1250-NEXT: v_dual_mov_b32 v13, s73 :: v_dual_mov_b32 v14, s60
; GFX1250-NEXT: v_mov_b32_e32 v15, s61
; GFX1250-NEXT: s_bfe_i64 s[60:61], s[64:65], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
; GFX1250-NEXT: s_lshr_b32 s38, s5, 18
; GFX1250-NEXT: s_lshr_b32 s16, s5, 19
-; GFX1250-NEXT: s_lshr_b32 s10, s5, 12
-; GFX1250-NEXT: s_lshr_b32 s12, s5, 13
+; GFX1250-NEXT: s_lshr_b32 s26, s5, 6
+; GFX1250-NEXT: s_lshr_b32 s30, s5, 7
+; GFX1250-NEXT: s_lshr_b32 s34, s5, 4
+; GFX1250-NEXT: s_lshr_b32 s36, s5, 5
+; GFX1250-NEXT: v_dual_mov_b32 v5, s97 :: v_dual_mov_b32 v6, s76
+; GFX1250-NEXT: v_dual_mov_b32 v7, s77 :: v_dual_mov_b32 v10, s86
; GFX1250-NEXT: v_dual_mov_b32 v11, s87 :: v_dual_mov_b32 v12, s72
; GFX1250-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[42:43], s[42:43], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v33, s3 :: v_dual_mov_b32 v34, s6
+; GFX1250-NEXT: v_dual_mov_b32 v35, s7 :: v_dual_mov_b32 v36, s8
+; GFX1250-NEXT: v_dual_mov_b32 v37, s9 :: v_dual_mov_b32 v38, s10
; GFX1250-NEXT: s_lshr_b32 s24, s5, 16
-; GFX1250-NEXT: s_lshr_b32 s14, s5, 10
-; GFX1250-NEXT: s_lshr_b32 s18, s5, 11
+; GFX1250-NEXT: s_lshr_b32 s40, s5, 2
+; GFX1250-NEXT: s_lshr_b32 s44, s5, 3
; GFX1250-NEXT: v_dual_mov_b32 v16, s60 :: v_dual_mov_b32 v17, s61
; GFX1250-NEXT: v_mov_b32_e32 v18, s54
; GFX1250-NEXT: s_bfe_i64 s[50:51], s[50:51], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s20, s5, 8
-; GFX1250-NEXT: s_lshr_b32 s22, s5, 9
-; GFX1250-NEXT: v_dual_mov_b32 v19, s55 :: v_dual_mov_b32 v20, s42
-; GFX1250-NEXT: v_dual_mov_b32 v21, s43 :: v_dual_mov_b32 v22, s50
-; GFX1250-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s26, s5, 6
-; GFX1250-NEXT: s_lshr_b32 s30, s5, 7
-; GFX1250-NEXT: v_dual_mov_b32 v23, s51 :: v_dual_mov_b32 v24, s28
-; GFX1250-NEXT: v_dual_mov_b32 v25, s29 :: v_dual_mov_b32 v26, s38
-; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:496
-; GFX1250-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:480
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v33, s3 :: v_dual_mov_b32 v0, s6
-; GFX1250-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s8
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v4, s10
-; GFX1250-NEXT: s_lshr_b32 s34, s5, 4
-; GFX1250-NEXT: s_lshr_b32 s36, s5, 5
-; GFX1250-NEXT: s_lshr_b32 s40, s5, 2
-; GFX1250-NEXT: s_lshr_b32 s44, s5, 3
-; GFX1250-NEXT: v_dual_mov_b32 v27, s39 :: v_dual_mov_b32 v28, s16
-; GFX1250-NEXT: v_dual_mov_b32 v29, s17 :: v_dual_mov_b32 v30, s24
; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s11 :: v_dual_mov_b32 v6, s12
-; GFX1250-NEXT: v_mov_b32_e32 v7, s13
+; GFX1250-NEXT: v_dual_mov_b32 v39, s11 :: v_dual_mov_b32 v40, s12
+; GFX1250-NEXT: v_dual_mov_b32 v41, s13 :: v_dual_mov_b32 v42, s14
; GFX1250-NEXT: s_lshr_b32 s46, s5, 1
; GFX1250-NEXT: s_mov_b32 s48, s5
-; GFX1250-NEXT: v_dual_mov_b32 v31, s25 :: v_dual_mov_b32 v32, s2
+; GFX1250-NEXT: v_dual_mov_b32 v19, s55 :: v_dual_mov_b32 v20, s42
+; GFX1250-NEXT: v_dual_mov_b32 v21, s43 :: v_dual_mov_b32 v22, s50
+; GFX1250-NEXT: s_bfe_i64 s[38:39], s[38:39], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v8, v[10:13], s[0:1] offset:464
-; GFX1250-NEXT: global_store_b128 v8, v[14:17], s[0:1] offset:448
-; GFX1250-NEXT: global_store_b128 v8, v[18:21], s[0:1] offset:432
-; GFX1250-NEXT: global_store_b128 v8, v[22:25], s[0:1] offset:416
-; GFX1250-NEXT: global_store_b128 v8, v[26:29], s[0:1] offset:400
-; GFX1250-NEXT: global_store_b128 v8, v[30:33], s[0:1] offset:384
-; GFX1250-NEXT: s_wait_xcnt 0x5
-; GFX1250-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v11, s15
-; GFX1250-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v13, s19
-; GFX1250-NEXT: s_wait_xcnt 0x4
-; GFX1250-NEXT: v_mov_b32_e32 v14, s20
+; GFX1250-NEXT: v_dual_mov_b32 v43, s15 :: v_dual_mov_b32 v44, s18
+; GFX1250-NEXT: v_dual_mov_b32 v45, s19 :: v_dual_mov_b32 v46, s20
; GFX1250-NEXT: s_lshr_b32 s52, s4, 30
; GFX1250-NEXT: s_lshr_b32 s56, s4, 31
+; GFX1250-NEXT: v_dual_mov_b32 v23, s51 :: v_dual_mov_b32 v24, s28
+; GFX1250-NEXT: v_dual_mov_b32 v25, s29 :: v_dual_mov_b32 v26, s38
+; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[44:45], s[44:45], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[40:41], s[40:41], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[36:37], s[36:37], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v15, s21 :: v_dual_mov_b32 v16, s22
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v17, s23 :: v_dual_mov_b32 v18, s26
+; GFX1250-NEXT: v_dual_mov_b32 v47, s21 :: v_dual_mov_b32 v48, s22
+; GFX1250-NEXT: v_dual_mov_b32 v49, s23 :: v_dual_mov_b32 v50, s26
; GFX1250-NEXT: s_lshr_b32 s58, s4, 28
; GFX1250-NEXT: s_lshr_b32 s62, s4, 29
+; GFX1250-NEXT: v_dual_mov_b32 v27, s39 :: v_dual_mov_b32 v28, s16
+; GFX1250-NEXT: v_dual_mov_b32 v29, s17 :: v_dual_mov_b32 v30, s24
; GFX1250-NEXT: s_bfe_i64 s[48:49], s[48:49], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[46:47], s[46:47], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v19, s27 :: v_dual_mov_b32 v20, s30
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v21, s31 :: v_dual_mov_b32 v22, s34
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:496
+; GFX1250-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:480
+; GFX1250-NEXT: v_dual_mov_b32 v51, s27 :: v_dual_mov_b32 v52, s30
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_dual_mov_b32 v53, s31 :: v_dual_mov_b32 v0, s34
+; GFX1250-NEXT: v_dual_mov_b32 v1, s35 :: v_dual_mov_b32 v2, s36
+; GFX1250-NEXT: v_mov_b32_e32 v3, s37
; GFX1250-NEXT: s_lshr_b32 s66, s4, 26
; GFX1250-NEXT: s_lshr_b32 s68, s4, 27
+; GFX1250-NEXT: v_dual_mov_b32 v31, s25 :: v_dual_mov_b32 v32, s2
; GFX1250-NEXT: s_bfe_i64 s[56:57], s[56:57], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v23, s35 :: v_dual_mov_b32 v24, s36
-; GFX1250-NEXT: v_mov_b32_e32 v25, s37
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:368
-; GFX1250-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:352
-; GFX1250-NEXT: global_store_b128 v8, v[10:13], s[0:1] offset:336
-; GFX1250-NEXT: global_store_b128 v8, v[14:17], s[0:1] offset:320
-; GFX1250-NEXT: global_store_b128 v8, v[18:21], s[0:1] offset:304
-; GFX1250-NEXT: global_store_b128 v8, v[22:25], s[0:1] offset:288
-; GFX1250-NEXT: s_wait_xcnt 0x5
+; GFX1250-NEXT: s_clause 0xb
+; GFX1250-NEXT: global_store_b128 v8, v[10:13], s[0:1] offset:464
+; GFX1250-NEXT: global_store_b128 v8, v[14:17], s[0:1] offset:448
+; GFX1250-NEXT: global_store_b128 v8, v[18:21], s[0:1] offset:432
+; GFX1250-NEXT: global_store_b128 v8, v[22:25], s[0:1] offset:416
+; GFX1250-NEXT: global_store_b128 v8, v[26:29], s[0:1] offset:400
+; GFX1250-NEXT: global_store_b128 v8, v[30:33], s[0:1] offset:384
+; GFX1250-NEXT: global_store_b128 v8, v[34:37], s[0:1] offset:368
+; GFX1250-NEXT: global_store_b128 v8, v[38:41], s[0:1] offset:352
+; GFX1250-NEXT: global_store_b128 v8, v[42:45], s[0:1] offset:336
+; GFX1250-NEXT: global_store_b128 v8, v[46:49], s[0:1] offset:320
+; GFX1250-NEXT: global_store_b128 v8, v[50:53], s[0:1] offset:304
+; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:288
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v1, s41
; GFX1250-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s45
-; GFX1250-NEXT: s_wait_xcnt 0x4
; GFX1250-NEXT: v_mov_b32_e32 v4, s48
; GFX1250-NEXT: s_lshr_b32 s70, s4, 24
; GFX1250-NEXT: s_lshr_b32 s74, s4, 25
@@ -12166,14 +12132,12 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i64 s[62:63], s[62:63], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x10000
; GFX1250-NEXT: v_dual_mov_b32 v5, s49 :: v_dual_mov_b32 v6, s46
-; GFX1250-NEXT: s_wait_xcnt 0x3
; GFX1250-NEXT: v_dual_mov_b32 v7, s47 :: v_dual_mov_b32 v10, s52
; GFX1250-NEXT: s_lshr_b32 s82, s4, 20
; GFX1250-NEXT: s_lshr_b32 s84, s4, 21
; GFX1250-NEXT: s_bfe_i64 s[68:69], s[68:69], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[66:67], s[66:67], 0x10000
; GFX1250-NEXT: v_dual_mov_b32 v11, s53 :: v_dual_mov_b32 v12, s56
-; GFX1250-NEXT: s_wait_xcnt 0x2
; GFX1250-NEXT: v_dual_mov_b32 v13, s57 :: v_dual_mov_b32 v14, s58
; GFX1250-NEXT: s_lshr_b32 s88, s4, 18
; GFX1250-NEXT: s_lshr_b32 s92, s4, 19
@@ -12182,14 +12146,12 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_i64 s[74:75], s[74:75], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[70:71], s[70:71], 0x10000
; GFX1250-NEXT: v_dual_mov_b32 v15, s59 :: v_dual_mov_b32 v16, s62
-; GFX1250-NEXT: s_wait_xcnt 0x1
; GFX1250-NEXT: v_dual_mov_b32 v17, s63 :: v_dual_mov_b32 v18, s66
; GFX1250-NEXT: s_lshr_b32 s94, s4, 16
; GFX1250-NEXT: s_lshr_b32 s100, s4, 17
; GFX1250-NEXT: s_bfe_i64 s[84:85], s[84:85], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[82:83], s[82:83], 0x10000
; GFX1250-NEXT: v_dual_mov_b32 v19, s67 :: v_dual_mov_b32 v20, s68
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v21, s69 :: v_dual_mov_b32 v22, s70
; GFX1250-NEXT: s_lshr_b32 s98, s4, 14
; GFX1250-NEXT: s_lshr_b32 s102, s4, 15
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 665f84001a5f52..df02766b0bae9b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -3595,7 +3595,7 @@ define amdgpu_kernel void @constant_sextload_v16i32_to_v16i64(ptr addrspace(1) %
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b512 s[0:15], s[18:19], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v0, s14
+; GFX1250-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v0, s14
; GFX1250-NEXT: s_ashr_i32 s28, s11, 31
; GFX1250-NEXT: s_ashr_i32 s29, s10, 31
; GFX1250-NEXT: s_ashr_i32 s30, s13, 31
@@ -3625,20 +3625,19 @@ define amdgpu_kernel void @constant_sextload_v16i32_to_v16i64(ptr addrspace(1) %
; GFX1250-NEXT: s_ashr_i32 s19, s0, 31
; GFX1250-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v24, s2
; GFX1250-NEXT: v_dual_mov_b32 v23, s22 :: v_dual_mov_b32 v25, s21
-; GFX1250-NEXT: v_mov_b32_e32 v27, s20
+; GFX1250-NEXT: v_dual_mov_b32 v27, s20 :: v_dual_mov_b32 v28, s0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b128 v28, v[0:3], s[16:17] offset:112
-; GFX1250-NEXT: global_store_b128 v28, v[4:7], s[16:17] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19
-; GFX1250-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s18
+; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[16:17] offset:112
+; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[16:17] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v29, s19 :: v_dual_mov_b32 v30, s1
+; GFX1250-NEXT: v_mov_b32_e32 v31, s18
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v28, v[8:11], s[16:17] offset:80
-; GFX1250-NEXT: global_store_b128 v28, v[12:15], s[16:17] offset:64
-; GFX1250-NEXT: global_store_b128 v28, v[16:19], s[16:17] offset:48
-; GFX1250-NEXT: global_store_b128 v28, v[20:23], s[16:17] offset:32
-; GFX1250-NEXT: global_store_b128 v28, v[24:27], s[16:17] offset:16
-; GFX1250-NEXT: global_store_b128 v28, v[0:3], s[16:17]
+; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[16:17] offset:80
+; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[16:17] offset:64
+; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[16:17] offset:48
+; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[16:17] offset:32
+; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[16:17] offset:16
+; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[16:17]
; GFX1250-NEXT: s_endpgm
%ld = load <16 x i32>, ptr addrspace(4) %in
%ext = sext <16 x i32> %ld to <16 x i64>
@@ -4909,104 +4908,96 @@ define amdgpu_kernel void @constant_sextload_v32i32_to_v32i64(ptr addrspace(1) %
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b512 s[0:15], s[38:39], 0x0 nv
; GFX1250-NEXT: s_load_b512 s[16:31], s[38:39], 0x40 nv
-; GFX1250-NEXT: v_mov_b32_e32 v24, 0
+; GFX1250-NEXT: v_mov_b32_e32 v52, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s49, s15, 31
+; GFX1250-NEXT: s_ashr_i32 s45, s11, 31
+; GFX1250-NEXT: s_ashr_i32 s60, s27, 31
; GFX1250-NEXT: s_ashr_i32 s64, s31, 31
; GFX1250-NEXT: s_ashr_i32 s65, s30, 31
; GFX1250-NEXT: s_ashr_i32 s62, s29, 31
; GFX1250-NEXT: s_ashr_i32 s63, s28, 31
-; GFX1250-NEXT: s_ashr_i32 s60, s27, 31
+; GFX1250-NEXT: s_ashr_i32 s46, s10, 31
+; GFX1250-NEXT: s_ashr_i32 s47, s13, 31
+; GFX1250-NEXT: s_ashr_i32 s57, s23, 31
+; GFX1250-NEXT: s_ashr_i32 s58, s25, 31
+; GFX1250-NEXT: s_ashr_i32 s59, s24, 31
; GFX1250-NEXT: s_ashr_i32 s61, s26, 31
; GFX1250-NEXT: v_dual_mov_b32 v0, s30 :: v_dual_mov_b32 v2, s31
; GFX1250-NEXT: v_dual_mov_b32 v4, s28 :: v_dual_mov_b32 v1, s65
-; GFX1250-NEXT: v_mov_b32_e32 v3, s64
-; GFX1250-NEXT: s_ashr_i32 s58, s25, 31
-; GFX1250-NEXT: s_ashr_i32 s59, s24, 31
-; GFX1250-NEXT: v_dual_mov_b32 v6, s29 :: v_dual_mov_b32 v8, s26
-; GFX1250-NEXT: v_dual_mov_b32 v5, s63 :: v_dual_mov_b32 v7, s62
-; GFX1250-NEXT: v_dual_mov_b32 v9, s61 :: v_dual_mov_b32 v10, s27
-; GFX1250-NEXT: v_dual_mov_b32 v11, s60 :: v_dual_mov_b32 v12, s24
-; GFX1250-NEXT: s_ashr_i32 s57, s23, 31
-; GFX1250-NEXT: v_dual_mov_b32 v13, s59 :: v_dual_mov_b32 v14, s25
-; GFX1250-NEXT: v_mov_b32_e32 v15, s58
+; GFX1250-NEXT: v_dual_mov_b32 v3, s64 :: v_dual_mov_b32 v11, s60
+; GFX1250-NEXT: v_mov_b32_e32 v12, s24
; GFX1250-NEXT: s_ashr_i32 s24, s22, 31
-; GFX1250-NEXT: s_ashr_i32 s55, s21, 31
-; GFX1250-NEXT: s_ashr_i32 s56, s20, 31
-; GFX1250-NEXT: s_ashr_i32 s53, s19, 31
-; GFX1250-NEXT: s_ashr_i32 s54, s18, 31
-; GFX1250-NEXT: s_clause 0x3
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:240
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:224
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:208
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:192
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v0, s22 :: v_dual_mov_b32 v1, s24
-; GFX1250-NEXT: v_dual_mov_b32 v2, s23 :: v_dual_mov_b32 v3, s57
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_mov_b32_e32 v4, s20
-; GFX1250-NEXT: s_ashr_i32 s51, s17, 31
-; GFX1250-NEXT: s_ashr_i32 s52, s16, 31
-; GFX1250-NEXT: v_dual_mov_b32 v5, s56 :: v_dual_mov_b32 v6, s21
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v7, s55 :: v_dual_mov_b32 v8, s18
-; GFX1250-NEXT: s_ashr_i32 s50, s14, 31
-; GFX1250-NEXT: v_dual_mov_b32 v9, s54 :: v_dual_mov_b32 v10, s19
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v11, s53 :: v_dual_mov_b32 v12, s16
-; GFX1250-NEXT: s_ashr_i32 s45, s11, 31
-; GFX1250-NEXT: s_ashr_i32 s46, s10, 31
-; GFX1250-NEXT: s_ashr_i32 s47, s13, 31
-; GFX1250-NEXT: s_ashr_i32 s48, s12, 31
-; GFX1250-NEXT: v_dual_mov_b32 v13, s52 :: v_dual_mov_b32 v14, s17
-; GFX1250-NEXT: v_dual_mov_b32 v15, s51 :: v_dual_mov_b32 v16, s14
; GFX1250-NEXT: s_ashr_i32 s43, s9, 31
; GFX1250-NEXT: s_ashr_i32 s44, s8, 31
-; GFX1250-NEXT: v_dual_mov_b32 v17, s50 :: v_dual_mov_b32 v18, s15
-; GFX1250-NEXT: v_dual_mov_b32 v19, s49 :: v_dual_mov_b32 v20, s12
+; GFX1250-NEXT: s_ashr_i32 s55, s21, 31
+; GFX1250-NEXT: s_ashr_i32 s56, s20, 31
+; GFX1250-NEXT: v_dual_mov_b32 v6, s29 :: v_dual_mov_b32 v8, s26
+; GFX1250-NEXT: v_dual_mov_b32 v5, s63 :: v_dual_mov_b32 v7, s62
; GFX1250-NEXT: s_ashr_i32 s41, s7, 31
; GFX1250-NEXT: s_ashr_i32 s42, s6, 31
-; GFX1250-NEXT: v_dual_mov_b32 v21, s48 :: v_dual_mov_b32 v22, s13
-; GFX1250-NEXT: v_mov_b32_e32 v23, s47
-; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:176
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:160
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:144
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:128
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:112
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[36:37] offset:96
-; GFX1250-NEXT: s_wait_xcnt 0x5
-; GFX1250-NEXT: v_dual_mov_b32 v0, s10 :: v_dual_mov_b32 v1, s46
-; GFX1250-NEXT: v_dual_mov_b32 v2, s11 :: v_dual_mov_b32 v3, s45
-; GFX1250-NEXT: s_wait_xcnt 0x4
-; GFX1250-NEXT: v_mov_b32_e32 v4, s8
+; GFX1250-NEXT: s_ashr_i32 s53, s19, 31
+; GFX1250-NEXT: s_ashr_i32 s54, s18, 31
+; GFX1250-NEXT: v_dual_mov_b32 v9, s61 :: v_dual_mov_b32 v10, s27
+; GFX1250-NEXT: v_dual_mov_b32 v15, s58 :: v_dual_mov_b32 v16, s22
+; GFX1250-NEXT: v_dual_mov_b32 v17, s24 :: v_dual_mov_b32 v18, s23
+; GFX1250-NEXT: v_dual_mov_b32 v19, s57 :: v_dual_mov_b32 v20, s20
+; GFX1250-NEXT: v_dual_mov_b32 v39, s47 :: v_dual_mov_b32 v40, s10
+; GFX1250-NEXT: v_dual_mov_b32 v41, s46 :: v_dual_mov_b32 v42, s11
+; GFX1250-NEXT: v_dual_mov_b32 v43, s45 :: v_dual_mov_b32 v44, s8
; GFX1250-NEXT: s_ashr_i32 s39, s5, 31
; GFX1250-NEXT: s_ashr_i32 s40, s4, 31
-; GFX1250-NEXT: v_dual_mov_b32 v5, s44 :: v_dual_mov_b32 v6, s9
-; GFX1250-NEXT: s_wait_xcnt 0x3
-; GFX1250-NEXT: v_dual_mov_b32 v7, s43 :: v_dual_mov_b32 v8, s6
+; GFX1250-NEXT: s_ashr_i32 s51, s17, 31
+; GFX1250-NEXT: s_ashr_i32 s52, s16, 31
+; GFX1250-NEXT: v_dual_mov_b32 v13, s59 :: v_dual_mov_b32 v14, s25
+; GFX1250-NEXT: v_dual_mov_b32 v21, s56 :: v_dual_mov_b32 v22, s21
+; GFX1250-NEXT: v_dual_mov_b32 v23, s55 :: v_dual_mov_b32 v24, s18
+; GFX1250-NEXT: v_dual_mov_b32 v45, s44 :: v_dual_mov_b32 v46, s9
+; GFX1250-NEXT: v_dual_mov_b32 v47, s43 :: v_dual_mov_b32 v48, s6
; GFX1250-NEXT: s_ashr_i32 s35, s3, 31
; GFX1250-NEXT: s_ashr_i32 s38, s2, 31
-; GFX1250-NEXT: v_dual_mov_b32 v9, s42 :: v_dual_mov_b32 v10, s7
-; GFX1250-NEXT: s_wait_xcnt 0x2
-; GFX1250-NEXT: v_dual_mov_b32 v11, s41 :: v_dual_mov_b32 v12, s4
+; GFX1250-NEXT: s_ashr_i32 s49, s15, 31
+; GFX1250-NEXT: s_ashr_i32 s50, s14, 31
+; GFX1250-NEXT: v_dual_mov_b32 v25, s54 :: v_dual_mov_b32 v26, s19
+; GFX1250-NEXT: v_dual_mov_b32 v27, s53 :: v_dual_mov_b32 v28, s16
+; GFX1250-NEXT: v_dual_mov_b32 v49, s42 :: v_dual_mov_b32 v50, s7
+; GFX1250-NEXT: v_mov_b32_e32 v51, s41
; GFX1250-NEXT: s_ashr_i32 s33, s1, 31
; GFX1250-NEXT: s_ashr_i32 s34, s0, 31
-; GFX1250-NEXT: v_dual_mov_b32 v13, s40 :: v_dual_mov_b32 v14, s5
+; GFX1250-NEXT: s_ashr_i32 s48, s12, 31
+; GFX1250-NEXT: s_clause 0x3
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[36:37] offset:240
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[36:37] offset:224
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[36:37] offset:208
+; GFX1250-NEXT: global_store_b128 v52, v[12:15], s[36:37] offset:192
+; GFX1250-NEXT: v_dual_mov_b32 v29, s52 :: v_dual_mov_b32 v30, s17
+; GFX1250-NEXT: v_dual_mov_b32 v31, s51 :: v_dual_mov_b32 v32, s14
+; GFX1250-NEXT: s_wait_xcnt 0x3
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s40
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s39
+; GFX1250-NEXT: s_wait_xcnt 0x2
+; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v33, s50
+; GFX1250-NEXT: v_dual_mov_b32 v34, s15 :: v_dual_mov_b32 v35, s49
+; GFX1250-NEXT: v_dual_mov_b32 v36, s12 :: v_dual_mov_b32 v5, s38
+; GFX1250-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s35
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v15, s39 :: v_dual_mov_b32 v16, s2
-; GFX1250-NEXT: v_dual_mov_b32 v17, s38 :: v_dual_mov_b32 v18, s3
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v19, s35 :: v_dual_mov_b32 v20, s0
-; GFX1250-NEXT: v_dual_mov_b32 v21, s34 :: v_dual_mov_b32 v22, s1
-; GFX1250-NEXT: v_mov_b32_e32 v23, s33
+; GFX1250-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v37, s48
+; GFX1250-NEXT: v_mov_b32_e32 v38, s13
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v52, v[16:19], s[36:37] offset:176
+; GFX1250-NEXT: global_store_b128 v52, v[20:23], s[36:37] offset:160
+; GFX1250-NEXT: global_store_b128 v52, v[24:27], s[36:37] offset:144
+; GFX1250-NEXT: global_store_b128 v52, v[28:31], s[36:37] offset:128
+; GFX1250-NEXT: global_store_b128 v52, v[32:35], s[36:37] offset:112
+; GFX1250-NEXT: global_store_b128 v52, v[36:39], s[36:37] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v9, s34 :: v_dual_mov_b32 v10, s1
+; GFX1250-NEXT: v_mov_b32_e32 v11, s33
; GFX1250-NEXT: s_clause 0x5
-; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[36:37] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[36:37] offset:64
-; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[36:37] offset:48
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[36:37] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[36:37] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[36:37]
+; GFX1250-NEXT: global_store_b128 v52, v[40:43], s[36:37] offset:80
+; GFX1250-NEXT: global_store_b128 v52, v[44:47], s[36:37] offset:64
+; GFX1250-NEXT: global_store_b128 v52, v[48:51], s[36:37] offset:48
+; GFX1250-NEXT: global_store_b128 v52, v[0:3], s[36:37] offset:32
+; GFX1250-NEXT: global_store_b128 v52, v[4:7], s[36:37] offset:16
+; GFX1250-NEXT: global_store_b128 v52, v[8:11], s[36:37]
; GFX1250-NEXT: s_endpgm
%ld = load <32 x i32>, ptr addrspace(4) %in
%ext = sext <32 x i32> %ld to <32 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll
index 17f85f02e73334..61561b9c791d6f 100644
--- a/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll
@@ -108,10 +108,10 @@ define i64 @add_u64_vv(i64 %v, i64 %a) {
define amdgpu_kernel void @add_u64_sv(i64 %v) {
; GCN-LABEL: add_u64_sv:
; GFX942: v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX1250: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250: v_add_nc_u64_e32 v[{{[0-9:]+}}], s[0:1], v[0:1]
; GISEL-LABEL: add_u64_sv:
; GFX942-GISEL: v_add_co_u32_e32 v{{[0-9:]+}}, vcc, s{{[0-9:]+}}, v{{[0-9:]+}}
-; GFX1250-GISEL: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-GISEL: v_add_nc_u64_e32 v[{{[0-9:]+}}], s[0:1], v[0:1]
%a = load i64, ptr poison
%add = add i64 %v, %a
store i64 %add, ptr poison
@@ -121,10 +121,10 @@ define amdgpu_kernel void @add_u64_sv(i64 %v) {
define amdgpu_kernel void @add_u64_vs(i64 %a) {
; GCN-LABEL: add_u64_vs:
; GFX942: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX1250: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250: v_add_nc_u64_e32 v[{{[0-9:]+}}], s[0:1], v[0:1]
; GISEL-LABEL: add_u64_vs:
; GFX942-GISEL: v_add_co_u32_e32 v{{[0-9:]+}}, vcc, s{{[0-9:]+}}, v{{[0-9:]+}}
-; GFX1250-GISEL: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-GISEL: v_add_nc_u64_e32 v[{{[0-9:]+}}], s[0:1], v[0:1]
%v = load i64, ptr poison
%add = add i64 %v, %a
store i64 %add, ptr poison
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 5ad9ca723d3fdf..413306ac190160 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -42,8 +42,8 @@ define amdgpu_kernel void @v_test_imax_sge_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_i32_e32 v0, s2, v0
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_max_i32_e32 v2, s2, v0
+; GFX1250-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX1250-NEXT: s_endpgm
;
; EG-LABEL: v_test_imax_sge_i32:
@@ -109,10 +109,10 @@ define amdgpu_kernel void @v_test_imax_sge_v4i32(ptr addrspace(1) %out, ptr addr
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: v_mov_b32_e32 v4, 0
+; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-NEXT: v_mov_b32_e32 v5, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[0:3], v0, s[0:1] scale_offset
+; GFX1250-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -123,7 +123,7 @@ define amdgpu_kernel void @v_test_imax_sge_v4i32(ptr addrspace(1) %out, ptr addr
; GFX1250-NEXT: v_max_i32_e32 v2, s6, v2
; GFX1250-NEXT: v_max_i32_e32 v1, s5, v1
; GFX1250-NEXT: v_max_i32_e32 v0, s4, v0
-; GFX1250-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX1250-NEXT: global_store_b128 v5, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
;
; EG-LABEL: v_test_imax_sge_v4i32:
@@ -464,8 +464,8 @@ define amdgpu_kernel void @v_test_imax_sgt_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_i32_e32 v0, s2, v0
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_max_i32_e32 v2, s2, v0
+; GFX1250-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX1250-NEXT: s_endpgm
;
; EG-LABEL: v_test_imax_sgt_i32:
@@ -581,8 +581,8 @@ define amdgpu_kernel void @v_test_umax_uge_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_u32_e32 v0, s2, v0
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: v_max_u32_e32 v2, s2, v0
+; GFX1250-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX1250-NEXT: s_endpgm
;
; EG-LABEL: v_test_umax_uge_i32:
@@ -830,8 +830,8 @@ define amdgpu_kernel void @v_test_umax_ugt_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_u32_e32 v0, s6, v0
-; GFX1250-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX1250-NEXT: v_max_u32_e32 v2, s6, v0
+; GFX1250-NEXT: global_store_b32 v1, v2, s[2:3]
; GFX1250-NEXT: s_endpgm
;
; EG-LABEL: v_test_umax_ugt_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index cab1b8a45ec5a9..60bf81825d52bd 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -2449,14 +2449,13 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_load_b64 v[0:1], v4, s[2:3] scale_offset
; GFX1250-NEXT: global_load_b64 v[2:3], v4, s[6:7] scale_offset
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, 3, v4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_pk_min_u16 v1, v1, v3
; GFX1250-NEXT: v_pk_min_u16 v0, v0, v2
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b16 v4, v1, s[0:1] offset:4
-; GFX1250-NEXT: global_store_b32 v4, v0, s[0:1]
+; GFX1250-NEXT: global_store_b16 v5, v1, s[0:1] offset:4
+; GFX1250-NEXT: global_store_b32 v5, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr inbounds <3 x i16>, ptr addrspace(1) %a.ptr, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index f1354ddf6c78e6..35964669cfb110 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -63,8 +63,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2v(<2 x i64> %v) {
; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[4:7], v[8:9], v[0:3]
-; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX1251-NEXT: v_pk_lshl_add_u64 v[2:5], v[4:7], v[8:9], v[0:3]
+; GFX1251-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1251-NEXT: s_endpgm
%a = load <2 x i64>, ptr poison
%shl = shl <2 x i64> %v, <i64 2, i64 2>
@@ -90,8 +90,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_v2s(<2 x i64> %a) {
; GFX1251-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
; GFX1251-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[0:3], v[8:9], v[4:7]
-; GFX1251-NEXT: flat_store_b128 v[0:1], v[0:3]
+; GFX1251-NEXT: v_pk_lshl_add_u64 v[2:5], v[0:3], v[8:9], v[4:7]
+; GFX1251-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1251-NEXT: s_endpgm
%v = load <2 x i64>, ptr poison
%shl = shl <2 x i64> %v, <i64 2, i64 2>
@@ -133,12 +133,11 @@ define i32 @pk_lshl_add_u64_gep(<2 x ptr> %p, <2 x i64> %a) {
; GFX1251-NEXT: s_mov_b32 s0, 2
; GFX1251-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s0
-; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[4:7], v[8:9], v[0:3]
-; GFX1251-NEXT: flat_load_b32 v4, v[0:1]
-; GFX1251-NEXT: flat_load_b32 v5, v[2:3]
+; GFX1251-NEXT: v_pk_lshl_add_u64 v[2:5], v[4:7], v[8:9], v[0:3]
+; GFX1251-NEXT: flat_load_b32 v0, v[2:3]
+; GFX1251-NEXT: flat_load_b32 v1, v[4:5]
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1251-NEXT: s_wait_xcnt 0x1
-; GFX1251-NEXT: v_add_nc_u32_e32 v0, v4, v5
+; GFX1251-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%gep = getelementptr inbounds i32, <2 x ptr> %p, <2 x i64> %a
%gep0 = extractelement <2 x ptr> %gep, i32 0
@@ -161,12 +160,11 @@ define i32 @pk_lshl_add_u64_maybe_oob(<2 x ptr> %p, <2 x i32> %i) {
; GFX1251-NEXT: v_mov_b32_e32 v8, s0
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-NEXT: v_pk_lshl_add_u64 v[0:3], v[4:7], v[8:9], v[0:3]
-; GFX1251-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], 12
-; GFX1251-NEXT: flat_load_b32 v4, v[0:1]
-; GFX1251-NEXT: flat_load_b32 v5, v[2:3]
+; GFX1251-NEXT: v_pk_add_nc_u64 v[2:5], v[0:3], 12
+; GFX1251-NEXT: flat_load_b32 v0, v[2:3]
+; GFX1251-NEXT: flat_load_b32 v1, v[4:5]
; GFX1251-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1251-NEXT: s_wait_xcnt 0x1
-; GFX1251-NEXT: v_add_nc_u32_e32 v0, v4, v5
+; GFX1251-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX1251-NEXT: s_set_pc_i64 s[30:31]
%idx = add nsw <2 x i32> %i, <i32 3, i32 3>
%gep = getelementptr i32, <2 x ptr> %p, <2 x i32> %idx
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index c73e60022b6231..1ef54038fe5b60 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -1315,23 +1315,20 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX12-5-SDAG-NEXT: global_load_u16 v1, v[0:1], off scope:SCOPE_SYS
-; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
; GFX12-5-SDAG-NEXT: global_load_u16 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-5-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-5-SDAG-NEXT: global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
+; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-5-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX12-5-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-5-SDAG-NEXT: v_min_u32_e32 v3, v0, v1
-; GFX12-5-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_sub_nc_u32 v0, v0, v3
+; GFX12-5-SDAG-NEXT: v_min_u32_e32 v5, v4, v2
+; GFX12-5-SDAG-NEXT: v_max_u32_e32 v2, v4, v2
+; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_sub_nc_u32 v2, v2, v5
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-5-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX12-5-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v3
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX12-5-SDAG-NEXT: global_store_b16 v4, v2, s[0:1]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i16_pat2:
@@ -1344,29 +1341,28 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX12-5-GISEL-NEXT: global_load_u16 v1, v[0:1], off scope:SCOPE_SYS
+; GFX12-5-GISEL-NEXT: global_load_u16 v2, v[0:1], off scope:SCOPE_SYS
+; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-5-GISEL-NEXT: global_load_u16 v3, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-5-GISEL-NEXT: ; kill: killed $vgpr0_vgpr1
; GFX12-5-GISEL-NEXT: s_and_b32 s4, 0xffff, s2
-; GFX12-5-GISEL-NEXT: global_load_u16 v2, v[0:1], off scope:SCOPE_SYS
-; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s3, v2
; GFX12-5-GISEL-NEXT: s_and_b32 s5, 0xffff, s3
; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_cmp_gt_u32 s4, s5
+; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s6, s2, s3
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s2, s3, s2
; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s5, v2
+; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s2, s2, s5
-; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-5-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-5-GISEL-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX12-5-GISEL-NEXT: global_store_b16 v3, v2, s[0:1]
; GFX12-5-GISEL-NEXT: s_endpgm
%a = load volatile i16, ptr addrspace(1) poison
%b = load volatile i16, ptr addrspace(1) poison
@@ -1527,20 +1523,17 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-5-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX12-5-SDAG-NEXT: global_load_u8 v1, v[0:1], off scope:SCOPE_SYS
-; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
-; GFX12-5-SDAG-NEXT: ; kill: killed $vgpr0_vgpr1
; GFX12-5-SDAG-NEXT: global_load_u8 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-SDAG-NEXT: v_min_u32_e32 v3, v0, v1
-; GFX12-5-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX12-5-SDAG-NEXT: global_load_u8 v3, v[0:1], off scope:SCOPE_SYS
+; GFX12-5-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-5-SDAG-NEXT: v_min_u32_e32 v4, v0, v2
+; GFX12-5-SDAG-NEXT: v_max_u32_e32 v2, v0, v2
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-5-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_sub_nc_u32 v0, v0, v3
-; GFX12-5-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX12-5-SDAG-NEXT: v_dual_sub_nc_u32 v2, v2, v4 :: v_dual_mov_b32 v4, 0
+; GFX12-5-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v3
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-5-SDAG-NEXT: global_store_b8 v4, v2, s[0:1]
; GFX12-5-SDAG-NEXT: s_endpgm
;
; GFX12-5-GISEL-LABEL: v_sad_u32_i8_pat2:
@@ -1553,26 +1546,25 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-5-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX12-5-GISEL-NEXT: s_wait_xcnt 0x0
-; GFX12-5-GISEL-NEXT: global_load_u8 v1, v[0:1], off scope:SCOPE_SYS
-; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-5-GISEL-NEXT: ; kill: killed $vgpr0_vgpr1
; GFX12-5-GISEL-NEXT: global_load_u8 v2, v[0:1], off scope:SCOPE_SYS
; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s3, v1
-; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-5-GISEL-NEXT: global_load_u8 v3, v[0:1], off scope:SCOPE_SYS
+; GFX12-5-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX12-5-GISEL-NEXT: s_cmp_gt_u32 s2, s3
; GFX12-5-GISEL-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s6, s2, s3
; GFX12-5-GISEL-NEXT: s_sub_co_i32 s2, s3, s2
; GFX12-5-GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-5-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; GFX12-5-GISEL-NEXT: v_readfirstlane_b32 s5, v2
+; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-GISEL-NEXT: s_add_co_i32 s2, s2, s5
-; GFX12-5-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-5-GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX12-5-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-GISEL-NEXT: global_store_b8 v1, v0, s[0:1]
+; GFX12-5-GISEL-NEXT: global_store_b8 v3, v2, s[0:1]
; GFX12-5-GISEL-NEXT: s_endpgm
%a = load volatile i8, ptr addrspace(1) poison
%b = load volatile i8, ptr addrspace(1) poison
diff --git a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
index 02a8e925db2262..5c529ead94077b 100644
--- a/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
+++ b/llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll
@@ -326,8 +326,8 @@ define amdgpu_ps void @v_cos_bf16_mul2_denorm_ieee(bfloat %in, ptr addrspace(1)
; SDAG-FAKE16-NEXT: v_cos_bf16_e32 v0, v0
; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; SDAG-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v0
-; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: v_pk_add_bf16 v1, v0, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v1, off
; SDAG-FAKE16-NEXT: s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_denorm_ieee:
@@ -339,8 +339,8 @@ define amdgpu_ps void @v_cos_bf16_mul2_denorm_ieee(bfloat %in, ptr addrspace(1)
; SDAG-REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; SDAG-REAL16-NEXT: v_pk_add_bf16 v0, v0, v0
-; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: v_pk_add_bf16 v1, v0, v0
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v1, off
; SDAG-REAL16-NEXT: s_endpgm
%cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
%mul2 = fmul nsz bfloat %cos, 2.0
@@ -360,8 +360,8 @@ define amdgpu_ps void @v_cos_bf16_mul2_f32_denorm_flush(bfloat %in, ptr addrspac
; SDAG-FAKE16-NEXT: v_cos_bf16_e32 v0, v0
; SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; SDAG-FAKE16-NEXT: v_pk_add_bf16 v0, v0, v0
-; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-FAKE16-NEXT: v_pk_add_bf16 v1, v0, v0
+; SDAG-FAKE16-NEXT: global_store_b16 v[2:3], v1, off
; SDAG-FAKE16-NEXT: s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_f32_denorm_flush:
@@ -373,8 +373,8 @@ define amdgpu_ps void @v_cos_bf16_mul2_f32_denorm_flush(bfloat %in, ptr addrspac
; SDAG-REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
; SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; SDAG-REAL16-NEXT: v_pk_add_bf16 v0, v0, v0
-; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v0, off
+; SDAG-REAL16-NEXT: v_pk_add_bf16 v1, v0, v0
+; SDAG-REAL16-NEXT: global_store_b16 v[2:3], v1, off
; SDAG-REAL16-NEXT: s_endpgm
%cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
%mul2 = fmul nsz bfloat %cos, 2.0
diff --git a/llvm/test/CodeGen/AMDGPU/vopd3-imm-fold.ll b/llvm/test/CodeGen/AMDGPU/vopd3-imm-fold.ll
index ebee18351e99da..03273f27e03129 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd3-imm-fold.ll
+++ b/llvm/test/CodeGen/AMDGPU/vopd3-imm-fold.ll
@@ -25,9 +25,9 @@ define amdgpu_kernel void @mask_shared_by_two_pairs(ptr addrspace(1) %out, ptr a
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v4
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, 2, v4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[2:3], v0, s[2:3]
+; GFX1250-NEXT: global_load_b64 v[2:3], v5, s[2:3]
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 s2, 0xffff0000
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -219,9 +219,9 @@ define amdgpu_kernel void @mask_inline_constant(ptr addrspace(1) %out, ptr addrs
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v4
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, 2, v4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[2:3], v0, s[2:3]
+; GFX1250-NEXT: global_load_b64 v[2:3], v5, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_bitop2_b32 v1, 63, v2 bitop3:0x40
; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v3 :: v_dual_bitop2_b32 v3, 63, v3 bitop3:0x40
@@ -276,9 +276,9 @@ define amdgpu_kernel void @mask_optsize(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v4
+; GFX1250-NEXT: v_lshlrev_b32_e32 v5, 2, v4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[2:3], v0, s[2:3]
+; GFX1250-NEXT: global_load_b64 v[2:3], v5, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -335,9 +335,9 @@ define amdgpu_kernel void @plain_vopd_component(ptr addrspace(1) %out, ptr addrs
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v2
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 2, v2
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3]
+; GFX1250-NEXT: global_load_b64 v[0:1], v3, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_dual_add_f32 v0, 0x3dcccccd, v0 :: v_dual_add_f32 v1, 0x3dcccccd, v1
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1] scale_offset
More information about the llvm-branch-commits
mailing list