[llvm] [AMDGPU] Avoid wmma bank conflict for GFX11 and GFX12 (PR #205530)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 01:32:09 PDT 2026
https://github.com/Shoreshen updated https://github.com/llvm/llvm-project/pull/205530
>From ce5f052a90e084a19c93b6578a0a0b56fa833f55 Mon Sep 17 00:00:00 2001
From: shore <372660931 at qq.com>
Date: Wed, 24 Jun 2026 19:17:12 +0800
Subject: [PATCH 1/6] avoid wmma bank conflict
---
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 67 +-
.../lib/Target/AMDGPU/SIMachineFunctionInfo.h | 25 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 106 ++-
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 9 +
.../AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll | 28 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 494 +++++++-------
.../llvm.amdgcn.sched.group.barrier.gfx11.ll | 206 +++---
.../CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll | 28 +-
llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll | 610 ++++++++++++++++++
...wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 20 +-
10 files changed, 1193 insertions(+), 400 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index 825634d7af65b..d4c2530afcd26 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -32,8 +32,10 @@
#include "GCNPreRAOptimizations.h"
#include "AMDGPU.h"
+#include "GCNRegPressure.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -54,6 +56,8 @@ class GCNPreRAOptimizationsImpl {
bool processReg(Register Reg);
void hintTrue16Copy(const MachineInstr &MI);
+ void recordWMMABankSiblings(const MachineInstr &MI,
+ SIMachineFunctionInfo &MFI);
bool optimizeBVHStack(MachineInstr &MI);
public:
@@ -262,6 +266,41 @@ void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
}
+void GCNPreRAOptimizationsImpl::recordWMMABankSiblings(
+ const MachineInstr &MI, SIMachineFunctionInfo &MFI) {
+ // The consecutive-allocation VGPR bank collision only happens when each
+ // matrix operand is a multiple of 4 VGPRs (128 bits) wide; with narrower
+ // operands the start registers already fall on different banks. Restrict the
+ // hint to WMMAs whose matrix source operands are a multiple of 128 bits.
+ auto Is128BitMultiple = [&](AMDGPU::OpName Name) {
+ const MachineOperand *MO = TII->getNamedOperand(MI, Name);
+ return MO && MO->isReg() && MO->getReg() &&
+ TRI->getRegSizeInBits(MO->getReg(), *MRI) % 128 == 0;
+ };
+ if (!Is128BitMultiple(AMDGPU::OpName::src0) ||
+ !Is128BitMultiple(AMDGPU::OpName::src1))
+ return;
+
+ // Record, for each virtual register operand, the other operands of this WMMA
+ // so SIRegisterInfo::getRegAllocationHints can steer them onto distinct
+ // VGPR banks. A register used by several WMMAs accumulates the union of their
+ // siblings.
+ SmallVector<Register, 4> Regs;
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.getReg() && MO.getReg().isVirtual() &&
+ !is_contained(Regs, MO.getReg()))
+ Regs.push_back(MO.getReg());
+
+ DenseMap<Register, SmallVector<Register, 3>> &Siblings =
+ MFI.getWMMABankSiblings();
+ for (Register Reg : Regs) {
+ SmallVector<Register, 3> &S = Siblings[Reg];
+ for (Register Other : Regs)
+ if (Other != Reg && !is_contained(S, Other))
+ S.push_back(Other);
+ }
+}
+
bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
SmallVector<Register, 2> UseRegs;
@@ -321,10 +360,15 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
const bool HasBVHStack = ST.hasBVHDualAndBVH8Insts();
const bool HasRealTrue16 = ST.useRealTrue16Insts();
+ // On gfx11/gfx12 the operands of a WMMA should be placed on distinct VGPR
+ // banks to avoid an issue-latency penalty.
+ const bool HasWMMABankHint = ST.hasFeature(AMDGPU::FeatureGFX11) ||
+ ST.hasFeature(AMDGPU::FeatureGFX12);
- if (!HasRealTrue16 && !HasBVHStack)
+ if (!HasRealTrue16 && !HasBVHStack && !HasWMMABankHint)
return Changed;
+ SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// Add RA hints to improve True16 COPY elimination.
@@ -332,6 +376,11 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
hintTrue16Copy(MI);
continue;
}
+ // Record WMMA operand siblings for VGPR bank-conflict avoidance hints.
+ if (HasWMMABankHint && SIInstrInfo::isWMMA(MI)) {
+ recordWMMABankSiblings(MI, *MFI);
+ continue;
+ }
// Add implicit uses to avoid early wait on intersect ray instructions.
if (HasBVHStack &&
(MI.getOpcode() == AMDGPU::DS_BVH_STACK_RTN_B32 ||
@@ -343,5 +392,21 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
}
}
+ // If any bank-sensitive WMMA was tagged, record the function's peak ArchVGPR
+ // pressure (the actual VGPR demand). The bank hint anchors its reserved
+ // regions to this instead of the top of the whole VGPR file.
+ if (HasWMMABankHint && !MFI->getWMMABankSiblings().empty()) {
+ unsigned Peak = 0;
+ GCNUpwardRPTracker RPT(*LIS);
+ for (const MachineBasicBlock &MBB : MF) {
+ RPT.reset(*MRI, LIS->getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
+ for (const MachineInstr &MI : reverse(MBB)) {
+ RPT.recede(MI);
+ Peak = std::max(Peak, RPT.getPressure().getArchVGPRNum());
+ }
+ }
+ MFI->setWMMAPeakVGPRPressure(Peak);
+ }
+
return Changed;
}
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 1f43505650222..0717e48132aca 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -617,6 +617,19 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
// load/store is enabled.
IndexedMap<uint32_t, VGPRBlock2IndexFunctor> MaskForVGPRBlockOps;
+ // Maps each virtual register that is an operand of a bank-sensitive v_wmma_*
+ // (matrix sources a multiple of 128 bits) to that instruction's other operand
+ // registers. Populated once by GCNPreRAOptimizations and consumed by
+ // SIRegisterInfo::getRegAllocationHints for VGPR bank-conflict avoidance, so
+ // the hint hook does not rescan the function for every virtual register.
+ DenseMap<Register, SmallVector<Register, 3>> WMMABankSiblings;
+
+ // Peak ArchVGPR pressure of the function (the "actual demand"), computed once
+ // by GCNPreRAOptimizations. Used by the WMMA bank hint to anchor its reserved
+ // bank regions to the top of the real footprint instead of the top of the
+ // whole VGPR file, so it does not inflate the VGPR count of small kernels.
+ unsigned WMMAPeakVGPRPressure = 0;
+
private:
Register VGPRForAGPRCopy;
@@ -650,6 +663,18 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
return MaskForVGPRBlockOps.inBounds(RegisterBlock);
}
+ // Accessors for the WMMA bank-hint map (see WMMABankSiblings).
+ DenseMap<Register, SmallVector<Register, 3>> &getWMMABankSiblings() {
+ return WMMABankSiblings;
+ }
+ const DenseMap<Register, SmallVector<Register, 3>> &
+ getWMMABankSiblings() const {
+ return WMMABankSiblings;
+ }
+
+ unsigned getWMMAPeakVGPRPressure() const { return WMMAPeakVGPRPressure; }
+ void setWMMAPeakVGPRPressure(unsigned P) { WMMAPeakVGPRPressure = P; }
+
public:
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9700720f0373a..ad18d7c940ddc 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4144,8 +4144,110 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
return false;
}
default:
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
- VRM);
+ break;
+ }
+
+ bool BaseImplRetVal =
+ TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
+
+ // Append v_wmma_* bank-conflict avoidance candidates (gfx11/gfx12). This only
+ // augments the candidate order for VirtReg and never changes BaseImplRetVal.
+ addWMMABankConflictHints(VirtReg, Order, Hints, MF, VRM);
+
+ return BaseImplRetVal;
+}
+
+// VGPR bank-conflict avoidance for v_wmma_* operands on gfx11/gfx12.
+//
+// A WMMA reads src0/src1/src2 through per-bank read ports (bank = first VGPR
+// % 4); when two operands share a bank the instruction pays an extra
+// issue-latency cycle. GCNPreRAOptimizations records, for each operand of a
+// bank-sensitive WMMA, that instruction's other operands (WMMABankSiblings).
+// Here, for the siblings that already have a physreg, append the candidates
+// from a not-yet-used bank so the allocator prefers them.
+//
+// The bias is applied via the candidate list (not by recording an MRI
+// allocation hint) on purpose: an MRI hint feeds the allocation-priority
+// computation (VRM::hasKnownPreference) and can reorder allocation so the
+// result register displaces the live-in source operands, introducing copies.
+// Appending here happens after the copy hints emitted by the base
+// implementation, so coalescing keeps priority and only this register's
+// candidate order is affected (never the global allocation order).
+void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
+ ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &Hints,
+ const MachineFunction &MF,
+ const VirtRegMap *VRM) const {
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ if (!VRM || !isVGPR(MRI, VirtReg))
+ return;
+
+ const SIMachineFunctionInfo &FuncInfo = *MF.getInfo<SIMachineFunctionInfo>();
+ const DenseMap<Register, SmallVector<Register, 3>> &Siblings =
+ FuncInfo.getWMMABankSiblings();
+
+ auto It = Siblings.find(VirtReg);
+ if (It == Siblings.end())
+ return;
+
+ // Banks already taken by allocated siblings.
+ unsigned UsedBankMask = 0;
+ for (Register Sibling : It->second) {
+ Register Phys = Sibling;
+ if (Phys.isVirtual()) {
+ if (!VRM->hasPhys(Phys))
+ continue;
+ Phys = VRM->getPhys(Phys);
+ }
+ if (Phys.isPhysical() && isVGPR(MRI, Phys))
+ UsedBankMask |= 1u << (getHWRegIndex(Phys) % 4);
+ }
+
+ unsigned W = getRegSizeInBits(VirtReg, MRI) / 32;
+ if (W && (W % 4) != 0)
+ return;
+
+ // Build a hardware-index -> physreg map and record the highest start index.
+ // Order is not guaranteed to be sorted (non-kernel functions reorder their
+ // callee-saved VGPRs), so we look candidates up by index instead of trusting
+ // Order's sequence or its last element.
+ DenseMap<unsigned, MCPhysReg> IdxToReg;
+ unsigned MaxIdx = 0;
+ for (MCPhysReg PhysReg : Order) {
+ unsigned Idx = getHWRegIndex(PhysReg);
+ IdxToReg[Idx] = PhysReg;
+ MaxIdx = std::max(MaxIdx, Idx);
+ }
+
+ // Anchor the candidate window to the function's actual VGPR footprint: peak
+ // WMMA-region pressure, rounded up to the allocation granule and capped by
+ // the VGPR budget. Without this the window is the top of the whole VGPR file
+ // (~v256) for any function without an occupancy cap, so operands get steered
+ // to very high registers, inflating the footprint and dropping occupancy on
+ // low-pressure functions (and triggering MSG_DEALLOC_VGPRS).
+ unsigned MaxVGPR = MaxIdx + W;
+ if (unsigned Peak = FuncInfo.getWMMAPeakVGPRPressure()) {
+ unsigned Gran = AMDGPU::IsaInfo::getVGPRAllocGranule(
+ ST, FuncInfo.getDynamicVGPRBlockSize());
+ unsigned Want = alignTo(Peak, std::max(Gran, 1u));
+ if (unsigned Budget = ST.getMaxNumVGPRs(MF))
+ Want = std::min(Want, Budget);
+ if (Want && Want < MaxVGPR)
+ MaxVGPR = Want;
+ }
+ if (MaxVGPR < 4 * W + 3)
+ return;
+ unsigned N = (MaxVGPR - 3) / (4 * W); // blocks per bank region
+ unsigned BankSize = N * W;
+ for (unsigned n = 0; n < N; n++) {
+ for (unsigned Bank = 0; Bank < 4; Bank++) {
+ if (((UsedBankMask >> Bank) & 1) != 0)
+ continue;
+ unsigned Start = Bank * BankSize + Bank + n * W;
+ auto RIt = IdxToReg.find(Start);
+ if (RIt != IdxToReg.end())
+ Hints.push_back(RIt->second);
+ }
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 5e08e47ad4d83..ba633ba7c5dc1 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -364,6 +364,15 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
const MachineFunction &MF, const VirtRegMap *VRM,
const LiveRegMatrix *Matrix) const override;
+ /// Append physreg candidates that keep the v_wmma_* operand \p VirtReg in a
+ /// VGPR bank distinct from its already-allocated WMMA siblings (gfx11/gfx12).
+ /// Only augments \p Hints for \p VirtReg; the global allocation order is
+ /// unchanged.
+ void addWMMABankConflictHints(Register VirtReg, ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &Hints,
+ const MachineFunction &MF,
+ const VirtRegMap *VRM) const;
+
const int *getRegUnitPressureSets(MCRegUnit RegUnit) const override;
MCRegister getReturnAddressReg(const MachineFunction &MF) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
index 57d3db413a277..4a8b2d52b3925 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
@@ -95,16 +95,16 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_tied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_tied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_dual_mov_b32 v51, v39 :: v_dual_mov_b32 v50, v38
-; W32-NEXT: v_dual_mov_b32 v49, v37 :: v_dual_mov_b32 v48, v36
-; W32-NEXT: v_dual_mov_b32 v47, v35 :: v_dual_mov_b32 v46, v34
-; W32-NEXT: v_dual_mov_b32 v45, v33 :: v_dual_mov_b32 v44, v32
+; W32-NEXT: v_dual_mov_b32 v58, v39 :: v_dual_mov_b32 v57, v38
+; W32-NEXT: v_dual_mov_b32 v56, v37 :: v_dual_mov_b32 v55, v36
+; W32-NEXT: v_dual_mov_b32 v54, v35 :: v_dual_mov_b32 v53, v34
+; W32-NEXT: v_dual_mov_b32 v52, v33 :: v_dual_mov_b32 v51, v32
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[44:51]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[51:58]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
@@ -170,16 +170,16 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_tied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_tied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_dual_mov_b32 v51, v39 :: v_dual_mov_b32 v50, v38
-; W32-NEXT: v_dual_mov_b32 v49, v37 :: v_dual_mov_b32 v48, v36
-; W32-NEXT: v_dual_mov_b32 v47, v35 :: v_dual_mov_b32 v46, v34
-; W32-NEXT: v_dual_mov_b32 v45, v33 :: v_dual_mov_b32 v44, v32
+; W32-NEXT: v_dual_mov_b32 v58, v39 :: v_dual_mov_b32 v57, v38
+; W32-NEXT: v_dual_mov_b32 v56, v37 :: v_dual_mov_b32 v55, v36
+; W32-NEXT: v_dual_mov_b32 v54, v35 :: v_dual_mov_b32 v53, v34
+; W32-NEXT: v_dual_mov_b32 v52, v33 :: v_dual_mov_b32 v51, v32
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[44:51]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[51:58]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index ac121195de432..e3dfb6feab7c1 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -14,78 +14,74 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
+; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v34, v0
+; COEXEC-NEXT: v_dual_mov_b32 v35, v0 :: v_dual_mov_b32 v36, v0
+; COEXEC-NEXT: v_dual_mov_b32 v37, v0 :: v_dual_mov_b32 v38, v0
+; COEXEC-NEXT: v_dual_mov_b32 v39, v0 :: v_dual_mov_b32 v40, v0
+; COEXEC-NEXT: v_dual_mov_b32 v41, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
+; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v42, v0
+; COEXEC-NEXT: v_dual_mov_b32 v43, v0 :: v_dual_mov_b32 v44, v0
+; COEXEC-NEXT: v_dual_mov_b32 v45, v0 :: v_dual_mov_b32 v46, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v29, v0
+; COEXEC-NEXT: v_mov_b32_e32 v47, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v30, v0
+; COEXEC-NEXT: v_mov_b32_e32 v48, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
-; COEXEC-NEXT: v_mov_b32_e32 v31, v0
-; COEXEC-NEXT: v_cndmask_b32_e64 v32, 0, 1, s0
-; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v32
+; COEXEC-NEXT: v_mov_b32_e32 v49, v0
+; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
+; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB0_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v88, s2
+; COEXEC-NEXT: v_mov_b32_e32 v32, s2
; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v88 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v88
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v88 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v88 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v88 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v88 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v88 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v88 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v88 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v88 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v88 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v88 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v88 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v88 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v88 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v88 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v32 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v32
+; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v32 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v32 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[54:57], v32 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[50:53], v32 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[58:61], v32 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[62:65], v32 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[70:73], v32 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[66:69], v32 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[78:81], v32 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[74:77], v32 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[86:89], v32 offset:960
+; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v32 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v32 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v32 offset:768
; COEXEC-NEXT: s_wait_dscnt 0xc
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[24:31], v[16:23], v[42:49]
; COEXEC-NEXT: s_wait_dscnt 0x8
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[58:65], v[50:57], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[74:81], v[66:73], v[34:41]
; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[82:89], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[24:31], v[16:23], v[42:49]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[58:65], v[50:57], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[74:81], v[66:73], v[34:41]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[82:89], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB0_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v32, 0
+; COEXEC-NEXT: v_mov_b32_e32 v16, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; COEXEC-NEXT: global_store_b128 v16, v[46:49], s[0:1] offset:16
+; COEXEC-NEXT: global_store_b128 v16, v[42:45], s[0:1]
+; COEXEC-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
+; COEXEC-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
+; COEXEC-NEXT: global_store_b128 v16, v[38:41], s[0:1] offset:272
+; COEXEC-NEXT: global_store_b128 v16, v[34:37], s[0:1] offset:256
+; COEXEC-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
+; COEXEC-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
@@ -99,77 +95,73 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v34, v0
+; GCN-NEXT: v_dual_mov_b32 v35, v0 :: v_dual_mov_b32 v36, v0
+; GCN-NEXT: v_dual_mov_b32 v37, v0 :: v_dual_mov_b32 v38, v0
+; GCN-NEXT: v_dual_mov_b32 v39, v0 :: v_dual_mov_b32 v40, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
+; GCN-NEXT: v_dual_mov_b32 v41, v0 :: v_dual_mov_b32 v8, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
+; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; GCN-NEXT: s_xor_b32 s0, s0, -1
-; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0
-; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0
-; GCN-NEXT: v_mov_b32_e32 v26, v0
-; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24
-; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0
-; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0
-; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0
+; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
+; GCN-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
+; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v42, v0
+; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
+; GCN-NEXT: v_dual_mov_b32 v43, v0 :: v_dual_mov_b32 v44, v0
+; GCN-NEXT: v_dual_mov_b32 v45, v0 :: v_dual_mov_b32 v46, v0
+; GCN-NEXT: v_dual_mov_b32 v47, v0 :: v_dual_mov_b32 v48, v0
+; GCN-NEXT: v_mov_b32_e32 v49, v0
; GCN-NEXT: .LBB0_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v92, s2
+; GCN-NEXT: v_mov_b32_e32 v32, s2
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s2, s2, s1
-; GCN-NEXT: ds_load_tr16_b128 v[32:35], v92
-; GCN-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v92 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v92 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v92 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v92 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v92 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v92 offset:448
-; GCN-NEXT: ds_load_tr16_b128 v[64:67], v92 offset:512
-; GCN-NEXT: ds_load_tr16_b128 v[68:71], v92 offset:576
-; GCN-NEXT: ds_load_tr16_b128 v[72:75], v92 offset:640
-; GCN-NEXT: ds_load_tr16_b128 v[76:79], v92 offset:704
-; GCN-NEXT: ds_load_tr16_b128 v[80:83], v92 offset:768
-; GCN-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:832
-; GCN-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:896
-; GCN-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:960
+; GCN-NEXT: ds_load_tr16_b128 v[16:19], v32
+; GCN-NEXT: ds_load_tr16_b128 v[20:23], v32 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[24:27], v32 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[28:31], v32 offset:192
+; GCN-NEXT: ds_load_tr16_b128 v[50:53], v32 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[54:57], v32 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[58:61], v32 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[62:65], v32 offset:448
+; GCN-NEXT: ds_load_tr16_b128 v[66:69], v32 offset:512
+; GCN-NEXT: ds_load_tr16_b128 v[70:73], v32 offset:576
+; GCN-NEXT: ds_load_tr16_b128 v[74:77], v32 offset:640
+; GCN-NEXT: ds_load_tr16_b128 v[78:81], v32 offset:704
+; GCN-NEXT: ds_load_tr16_b128 v[82:85], v32 offset:768
+; GCN-NEXT: ds_load_tr16_b128 v[86:89], v32 offset:832
+; GCN-NEXT: ds_load_tr16_b128 v[90:93], v32 offset:896
+; GCN-NEXT: ds_load_tr16_b128 v[94:97], v32 offset:960
; GCN-NEXT: s_wait_dscnt 0xc
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[16:23], v[24:31], v[42:49]
; GCN-NEXT: s_wait_dscnt 0x8
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[50:57], v[58:65], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x4
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[66:73], v[74:81], v[34:41]
; GCN-NEXT: s_wait_dscnt 0x0
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[16:23], v[24:31], v[42:49]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[50:57], v[58:65], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[66:73], v[74:81], v[34:41]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v32, 0
+; GCN-NEXT: v_mov_b32_e32 v16, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; GCN-NEXT: global_store_b128 v16, v[46:49], s[0:1] offset:16
+; GCN-NEXT: global_store_b128 v16, v[42:45], s[0:1]
+; GCN-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
+; GCN-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
+; GCN-NEXT: global_store_b128 v16, v[38:41], s[0:1] offset:272
+; GCN-NEXT: global_store_b128 v16, v[34:37], s[0:1] offset:256
+; GCN-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
+; GCN-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
@@ -256,108 +248,104 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
+; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v66, v0
+; COEXEC-NEXT: v_dual_mov_b32 v67, v0 :: v_dual_mov_b32 v68, v0
+; COEXEC-NEXT: v_dual_mov_b32 v69, v0 :: v_dual_mov_b32 v70, v0
+; COEXEC-NEXT: v_dual_mov_b32 v71, v0 :: v_dual_mov_b32 v72, v0
+; COEXEC-NEXT: v_dual_mov_b32 v73, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
+; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v74, v0
+; COEXEC-NEXT: v_dual_mov_b32 v75, v0 :: v_dual_mov_b32 v76, v0
+; COEXEC-NEXT: v_dual_mov_b32 v77, v0 :: v_dual_mov_b32 v78, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v29, v0
+; COEXEC-NEXT: v_mov_b32_e32 v79, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v30, v0
+; COEXEC-NEXT: v_mov_b32_e32 v80, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
-; COEXEC-NEXT: v_mov_b32_e32 v31, v0
-; COEXEC-NEXT: v_cndmask_b32_e64 v32, 0, 1, s0
-; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v32
+; COEXEC-NEXT: v_mov_b32_e32 v81, v0
+; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
+; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB1_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_add_co_i32 s7, s2, s6
; COEXEC-NEXT: s_add_co_i32 s8, s3, s6
; COEXEC-NEXT: s_add_co_i32 s6, s6, s1
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v124, s7
+; COEXEC-NEXT: v_mov_b32_e32 v64, s7
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; COEXEC-NEXT: v_mov_b32_e32 v156, s8
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v124
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v124 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v156
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v156 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v124 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v124 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v156 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v156 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v124 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v124 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v156 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v156 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v124 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v124 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v156 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v156 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
+; COEXEC-NEXT: v_mov_b32_e32 v65, s8
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v64
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v64 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v65
+; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v65 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[86:89], v64 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v64 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v65 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v65 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v64 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v64 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v65 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v65 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v64 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v64 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v65 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v65 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[98:101], v64 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[102:105], v64 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[106:109], v64 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[110:113], v64 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[114:117], v64 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[118:121], v64 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[122:125], v64 offset:896
; COEXEC-NEXT: s_wait_dscnt 0x13
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[16:23], v[24:31], v[74:81]
+; COEXEC-NEXT: ds_load_tr16_b128 v[126:129], v64 offset:960
+; COEXEC-NEXT: ds_load_tr16_b128 v[130:133], v65 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[134:137], v65 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[138:141], v65 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[142:145], v65 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[146:149], v65 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[150:153], v65 offset:704
; COEXEC-NEXT: s_wait_dscnt 0x16
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[82:89], v[32:39], v[8:15]
+; COEXEC-NEXT: ds_load_tr16_b128 v[154:157], v65 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[158:161], v65 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x14
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[40:47], v[48:55], v[66:73]
; COEXEC-NEXT: s_wait_dscnt 0x10
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[56:63], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[16:23], v[24:31], v[74:81]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[82:89], v[32:39], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[40:47], v[48:55], v[66:73]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[56:63], v[0:7]
; COEXEC-NEXT: s_wait_dscnt 0x6
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[98:105], v[130:137], v[74:81]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[106:113], v[138:145], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x2
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[114:121], v[146:153], v[66:73]
; COEXEC-NEXT: s_wait_dscnt 0x0
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[122:129], v[154:161], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[98:105], v[130:137], v[74:81]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[106:113], v[138:145], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[114:121], v[146:153], v[66:73]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[122:129], v[154:161], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
-; COEXEC-NEXT: v_mov_b32_e32 v32, 0
+; COEXEC-NEXT: v_mov_b32_e32 v16, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; COEXEC-NEXT: global_store_b128 v16, v[78:81], s[0:1] offset:16
+; COEXEC-NEXT: global_store_b128 v16, v[74:77], s[0:1]
+; COEXEC-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
+; COEXEC-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
+; COEXEC-NEXT: global_store_b128 v16, v[70:73], s[0:1] offset:272
+; COEXEC-NEXT: global_store_b128 v16, v[66:69], s[0:1] offset:256
+; COEXEC-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
+; COEXEC-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
@@ -372,103 +360,103 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v50, v0
+; GCN-NEXT: v_dual_mov_b32 v51, v0 :: v_dual_mov_b32 v52, v0
+; GCN-NEXT: v_dual_mov_b32 v53, v0 :: v_dual_mov_b32 v54, v0
+; GCN-NEXT: v_dual_mov_b32 v55, v0 :: v_dual_mov_b32 v56, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
+; GCN-NEXT: v_dual_mov_b32 v57, v0 :: v_dual_mov_b32 v8, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
+; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; GCN-NEXT: s_xor_b32 s0, s0, -1
-; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0
-; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0
-; GCN-NEXT: v_mov_b32_e32 v26, v0
-; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24
-; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0
-; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0
-; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0
+; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
+; GCN-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
+; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v58, v0
+; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
+; GCN-NEXT: v_dual_mov_b32 v59, v0 :: v_dual_mov_b32 v60, v0
+; GCN-NEXT: v_dual_mov_b32 v61, v0 :: v_dual_mov_b32 v62, v0
+; GCN-NEXT: v_dual_mov_b32 v63, v0 :: v_dual_mov_b32 v64, v0
+; GCN-NEXT: v_mov_b32_e32 v65, v0
; GCN-NEXT: .LBB1_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: s_add_co_i32 s7, s2, s6
; GCN-NEXT: s_add_co_i32 s8, s3, s6
-; GCN-NEXT: v_dual_mov_b32 v96, s7 :: v_dual_mov_b32 v97, s8
+; GCN-NEXT: v_dual_mov_b32 v48, s7 :: v_dual_mov_b32 v49, s8
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s6, s6, s1
-; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96
-; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:512
-; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:576
-; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:512
-; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:576
-; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:768
-; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:832
-; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:768
-; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:832
+; GCN-NEXT: ds_load_tr16_b128 v[16:19], v48
+; GCN-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[24:27], v49
+; GCN-NEXT: ds_load_tr16_b128 v[28:31], v49 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[66:69], v48 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[70:73], v48 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[32:35], v49 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[36:39], v49 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v48 offset:512
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v48 offset:576
+; GCN-NEXT: ds_load_tr16_b128 v[74:77], v49 offset:512
+; GCN-NEXT: ds_load_tr16_b128 v[78:81], v49 offset:576
+; GCN-NEXT: ds_load_tr16_b128 v[82:85], v48 offset:768
+; GCN-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:832
+; GCN-NEXT: ds_load_tr16_b128 v[90:93], v49 offset:768
+; GCN-NEXT: ds_load_tr16_b128 v[94:97], v49 offset:832
; GCN-NEXT: s_wait_dscnt 0xc
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
; GCN-NEXT: s_wait_dscnt 0x8
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x4
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: s_wait_dscnt 0x0
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97 offset:128
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:448
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:384
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:448
-; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:640
-; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:704
-; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:640
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:704
-; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:896
-; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:960
-; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:896
-; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:960
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
+; GCN-NEXT: ds_load_tr16_b128 v[16:19], v48 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:192
+; GCN-NEXT: ds_load_tr16_b128 v[24:27], v49 offset:128
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
+; GCN-NEXT: ds_load_tr16_b128 v[28:31], v49 offset:192
+; GCN-NEXT: ds_load_tr16_b128 v[66:69], v48 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[70:73], v48 offset:448
+; GCN-NEXT: ds_load_tr16_b128 v[32:35], v49 offset:384
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
+; GCN-NEXT: ds_load_tr16_b128 v[36:39], v49 offset:448
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v48 offset:640
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v48 offset:704
+; GCN-NEXT: ds_load_tr16_b128 v[74:77], v49 offset:640
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
+; GCN-NEXT: ds_load_tr16_b128 v[78:81], v49 offset:704
+; GCN-NEXT: ds_load_tr16_b128 v[82:85], v48 offset:896
+; GCN-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:960
+; GCN-NEXT: ds_load_tr16_b128 v[90:93], v49 offset:896
+; GCN-NEXT: ds_load_tr16_b128 v[94:97], v49 offset:960
; GCN-NEXT: s_wait_dscnt 0xc
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
; GCN-NEXT: s_wait_dscnt 0x8
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x4
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: s_wait_dscnt 0x0
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v32, 0
+; GCN-NEXT: v_mov_b32_e32 v16, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; GCN-NEXT: global_store_b128 v16, v[62:65], s[0:1] offset:16
+; GCN-NEXT: global_store_b128 v16, v[58:61], s[0:1]
+; GCN-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
+; GCN-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
+; GCN-NEXT: global_store_b128 v16, v[54:57], s[0:1] offset:272
+; GCN-NEXT: global_store_b128 v16, v[50:53], s[0:1] offset:256
+; GCN-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
+; GCN-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
index 20523476a29d5..70283fa6504e8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
@@ -8,65 +8,62 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 5, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GCN-NEXT: v_and_b32_e32 v40, 0x7fe0, v0
+; GCN-NEXT: v_and_b32_e32 v16, 0x7fe0, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_nc_u32_e32 v32, s0, v40
-; GCN-NEXT: v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v40
-; GCN-NEXT: ds_load_b128 v[4:7], v32 offset:16
-; GCN-NEXT: ds_load_b128 v[12:15], v32 offset:2064
-; GCN-NEXT: ds_load_b128 v[20:23], v32 offset:6160
-; GCN-NEXT: ds_load_b128 v[28:31], v32 offset:12304
-; GCN-NEXT: ds_load_b128 v[36:39], v32 offset:20496
-; GCN-NEXT: ds_load_b128 v[0:3], v32
-; GCN-NEXT: ds_load_b128 v[8:11], v32 offset:2048
-; GCN-NEXT: ds_load_b128 v[16:19], v32 offset:6144
-; GCN-NEXT: ds_load_b128 v[24:27], v32 offset:12288
-; GCN-NEXT: ds_load_b128 v[32:35], v32 offset:20480
+; GCN-NEXT: v_add_nc_u32_e32 v8, s0, v16
+; GCN-NEXT: v_dual_mov_b32 v33, s1 :: v_dual_add_nc_u32 v16, s1, v16
+; GCN-NEXT: ds_load_b128 v[4:7], v8 offset:16
+; GCN-NEXT: ds_load_b128 v[21:24], v8 offset:2064
+; GCN-NEXT: ds_load_b128 v[38:41], v8 offset:6160
+; GCN-NEXT: ds_load_b128 v[55:58], v8 offset:12304
+; GCN-NEXT: ds_load_b128 v[12:15], v8 offset:20496
+; GCN-NEXT: ds_load_b128 v[0:3], v8
+; GCN-NEXT: ds_load_b128 v[17:20], v8 offset:2048
+; GCN-NEXT: ds_load_b128 v[34:37], v8 offset:6144
+; GCN-NEXT: ds_load_b128 v[51:54], v8 offset:12288
+; GCN-NEXT: ds_load_b128 v[8:11], v8 offset:20480
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)
-; GCN-NEXT: s_waitcnt lgkmcnt(4)
-; GCN-NEXT: v_mov_b32_e32 v47, v7
; GCN-NEXT: s_waitcnt lgkmcnt(3)
-; GCN-NEXT: v_mov_b32_e32 v55, v15
+; GCN-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v49, v24
; GCN-NEXT: s_waitcnt lgkmcnt(2)
-; GCN-NEXT: v_mov_b32_e32 v63, v23
+; GCN-NEXT: v_mov_b32_e32 v66, v41
; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mov_b32_e32 v71, v31
+; GCN-NEXT: v_mov_b32_e32 v74, v58
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v79, v39 :: v_dual_mov_b32 v46, v6
-; GCN-NEXT: v_dual_mov_b32 v45, v5 :: v_dual_mov_b32 v44, v4
-; GCN-NEXT: v_dual_mov_b32 v43, v3 :: v_dual_mov_b32 v42, v2
-; GCN-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v40, v0
-; GCN-NEXT: v_dual_mov_b32 v54, v14 :: v_dual_mov_b32 v53, v13
-; GCN-NEXT: v_dual_mov_b32 v52, v12 :: v_dual_mov_b32 v51, v11
-; GCN-NEXT: v_dual_mov_b32 v50, v10 :: v_dual_mov_b32 v49, v9
-; GCN-NEXT: v_mov_b32_e32 v48, v8
-; GCN-NEXT: v_dual_mov_b32 v62, v22 :: v_dual_mov_b32 v61, v21
-; GCN-NEXT: v_dual_mov_b32 v60, v20 :: v_dual_mov_b32 v59, v19
-; GCN-NEXT: v_dual_mov_b32 v58, v18 :: v_dual_mov_b32 v57, v17
-; GCN-NEXT: v_mov_b32_e32 v56, v16
-; GCN-NEXT: v_dual_mov_b32 v70, v30 :: v_dual_mov_b32 v69, v29
-; GCN-NEXT: v_dual_mov_b32 v68, v28 :: v_dual_mov_b32 v67, v27
-; GCN-NEXT: v_dual_mov_b32 v66, v26 :: v_dual_mov_b32 v65, v25
-; GCN-NEXT: v_mov_b32_e32 v64, v24
-; GCN-NEXT: v_dual_mov_b32 v78, v38 :: v_dual_mov_b32 v77, v37
-; GCN-NEXT: v_dual_mov_b32 v76, v36 :: v_dual_mov_b32 v75, v35
-; GCN-NEXT: v_dual_mov_b32 v74, v34 :: v_dual_mov_b32 v73, v33
-; GCN-NEXT: v_mov_b32_e32 v72, v32
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[40:47], v[0:7], v[0:7], v[40:47]
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[48:55], v[8:15], v[8:15], v[48:55]
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[56:63], v[16:23], v[16:23], v[56:63]
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[64:71], v[24:31], v[24:31], v[64:71]
-; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[72:79], v[32:39], v[32:39], v[72:79]
-; GCN-NEXT: ds_store_b128 v80, v[44:47] offset:16
-; GCN-NEXT: ds_store_b128 v80, v[40:43]
-; GCN-NEXT: ds_store_b128 v81, v[52:55] offset:2064
-; GCN-NEXT: ds_store_b128 v81, v[48:51] offset:2048
-; GCN-NEXT: ds_store_b128 v81, v[60:63] offset:4112
-; GCN-NEXT: ds_store_b128 v81, v[56:59] offset:4096
-; GCN-NEXT: ds_store_b128 v81, v[68:71] offset:6160
-; GCN-NEXT: ds_store_b128 v81, v[64:67] offset:6144
-; GCN-NEXT: ds_store_b128 v81, v[76:79] offset:8208
-; GCN-NEXT: ds_store_b128 v81, v[72:75] offset:8192
+; GCN-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v31, v6
+; GCN-NEXT: v_dual_mov_b32 v30, v5 :: v_dual_mov_b32 v29, v4
+; GCN-NEXT: v_dual_mov_b32 v28, v3 :: v_dual_mov_b32 v27, v2
+; GCN-NEXT: v_dual_mov_b32 v26, v1 :: v_dual_mov_b32 v25, v0
+; GCN-NEXT: v_dual_mov_b32 v48, v23 :: v_dual_mov_b32 v47, v22
+; GCN-NEXT: v_dual_mov_b32 v46, v21 :: v_dual_mov_b32 v45, v20
+; GCN-NEXT: v_dual_mov_b32 v44, v19 :: v_dual_mov_b32 v43, v18
+; GCN-NEXT: v_dual_mov_b32 v42, v17 :: v_dual_mov_b32 v65, v40
+; GCN-NEXT: v_dual_mov_b32 v64, v39 :: v_dual_mov_b32 v63, v38
+; GCN-NEXT: v_dual_mov_b32 v62, v37 :: v_dual_mov_b32 v61, v36
+; GCN-NEXT: v_dual_mov_b32 v60, v35 :: v_dual_mov_b32 v59, v34
+; GCN-NEXT: v_dual_mov_b32 v73, v57 :: v_dual_mov_b32 v72, v56
+; GCN-NEXT: v_dual_mov_b32 v71, v55 :: v_dual_mov_b32 v70, v54
+; GCN-NEXT: v_dual_mov_b32 v69, v53 :: v_dual_mov_b32 v68, v52
+; GCN-NEXT: v_mov_b32_e32 v67, v51
+; GCN-NEXT: v_dual_mov_b32 v81, v14 :: v_dual_mov_b32 v80, v13
+; GCN-NEXT: v_dual_mov_b32 v79, v12 :: v_dual_mov_b32 v78, v11
+; GCN-NEXT: v_dual_mov_b32 v77, v10 :: v_dual_mov_b32 v76, v9
+; GCN-NEXT: v_mov_b32_e32 v75, v8
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[25:32], v[0:7], v[0:7], v[25:32]
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[42:49], v[17:24], v[17:24], v[42:49]
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[59:66], v[34:41], v[34:41], v[59:66]
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[67:74], v[51:58], v[51:58], v[67:74]
+; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[75:82], v[8:15], v[8:15], v[75:82]
+; GCN-NEXT: ds_store_b128 v16, v[29:32] offset:16
+; GCN-NEXT: ds_store_b128 v16, v[25:28]
+; GCN-NEXT: ds_store_b128 v33, v[46:49] offset:2064
+; GCN-NEXT: ds_store_b128 v33, v[42:45] offset:2048
+; GCN-NEXT: ds_store_b128 v33, v[63:66] offset:4112
+; GCN-NEXT: ds_store_b128 v33, v[59:62] offset:4096
+; GCN-NEXT: ds_store_b128 v33, v[71:74] offset:6160
+; GCN-NEXT: ds_store_b128 v33, v[67:70] offset:6144
+; GCN-NEXT: ds_store_b128 v33, v[79:82] offset:8208
+; GCN-NEXT: ds_store_b128 v33, v[75:78] offset:8192
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(10) SyncID(0)
; GCN-NEXT: s_endpgm
@@ -76,65 +73,62 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 5, v0
; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; EXACTCUTOFF-NEXT: v_and_b32_e32 v40, 0x7fe0, v0
+; EXACTCUTOFF-NEXT: v_and_b32_e32 v16, 0x7fe0, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v32, s0, v40
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v40
-; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v32 offset:16
-; EXACTCUTOFF-NEXT: ds_load_b128 v[12:15], v32 offset:2064
-; EXACTCUTOFF-NEXT: ds_load_b128 v[20:23], v32 offset:6160
-; EXACTCUTOFF-NEXT: ds_load_b128 v[28:31], v32 offset:12304
-; EXACTCUTOFF-NEXT: ds_load_b128 v[36:39], v32 offset:20496
-; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v32
-; EXACTCUTOFF-NEXT: ds_load_b128 v[8:11], v32 offset:2048
-; EXACTCUTOFF-NEXT: ds_load_b128 v[16:19], v32 offset:6144
-; EXACTCUTOFF-NEXT: ds_load_b128 v[24:27], v32 offset:12288
-; EXACTCUTOFF-NEXT: ds_load_b128 v[32:35], v32 offset:20480
+; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v8, s0, v16
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v33, s1 :: v_dual_add_nc_u32 v16, s1, v16
+; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v8 offset:16
+; EXACTCUTOFF-NEXT: ds_load_b128 v[21:24], v8 offset:2064
+; EXACTCUTOFF-NEXT: ds_load_b128 v[38:41], v8 offset:6160
+; EXACTCUTOFF-NEXT: ds_load_b128 v[55:58], v8 offset:12304
+; EXACTCUTOFF-NEXT: ds_load_b128 v[12:15], v8 offset:20496
+; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v8
+; EXACTCUTOFF-NEXT: ds_load_b128 v[17:20], v8 offset:2048
+; EXACTCUTOFF-NEXT: ds_load_b128 v[34:37], v8 offset:6144
+; EXACTCUTOFF-NEXT: ds_load_b128 v[51:54], v8 offset:12288
+; EXACTCUTOFF-NEXT: ds_load_b128 v[8:11], v8 offset:20480
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(4)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v47, v7
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(3)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v55, v15
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v49, v24
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(2)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v63, v23
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v66, v41
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v71, v31
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v74, v58
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v79, v39 :: v_dual_mov_b32 v46, v6
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v45, v5 :: v_dual_mov_b32 v44, v4
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v43, v3 :: v_dual_mov_b32 v42, v2
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v40, v0
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v54, v14 :: v_dual_mov_b32 v53, v13
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v52, v12 :: v_dual_mov_b32 v51, v11
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v50, v10 :: v_dual_mov_b32 v49, v9
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v48, v8
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v62, v22 :: v_dual_mov_b32 v61, v21
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v60, v20 :: v_dual_mov_b32 v59, v19
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v58, v18 :: v_dual_mov_b32 v57, v17
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v56, v16
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v70, v30 :: v_dual_mov_b32 v69, v29
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v68, v28 :: v_dual_mov_b32 v67, v27
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v66, v26 :: v_dual_mov_b32 v65, v25
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v64, v24
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v78, v38 :: v_dual_mov_b32 v77, v37
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v76, v36 :: v_dual_mov_b32 v75, v35
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v74, v34 :: v_dual_mov_b32 v73, v33
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v72, v32
-; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[40:47], v[0:7], v[0:7], v[40:47]
-; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[48:55], v[8:15], v[8:15], v[48:55]
-; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[56:63], v[16:23], v[16:23], v[56:63]
-; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[64:71], v[24:31], v[24:31], v[64:71]
-; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[72:79], v[32:39], v[32:39], v[72:79]
-; EXACTCUTOFF-NEXT: ds_store_b128 v80, v[44:47] offset:16
-; EXACTCUTOFF-NEXT: ds_store_b128 v80, v[40:43]
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[52:55] offset:2064
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[48:51] offset:2048
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[60:63] offset:4112
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[56:59] offset:4096
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[68:71] offset:6160
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[64:67] offset:6144
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[76:79] offset:8208
-; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[72:75] offset:8192
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v31, v6
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v30, v5 :: v_dual_mov_b32 v29, v4
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v28, v3 :: v_dual_mov_b32 v27, v2
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v26, v1 :: v_dual_mov_b32 v25, v0
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v48, v23 :: v_dual_mov_b32 v47, v22
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v46, v21 :: v_dual_mov_b32 v45, v20
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v44, v19 :: v_dual_mov_b32 v43, v18
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v42, v17 :: v_dual_mov_b32 v65, v40
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v64, v39 :: v_dual_mov_b32 v63, v38
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v62, v37 :: v_dual_mov_b32 v61, v36
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v60, v35 :: v_dual_mov_b32 v59, v34
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v73, v57 :: v_dual_mov_b32 v72, v56
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v71, v55 :: v_dual_mov_b32 v70, v54
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v69, v53 :: v_dual_mov_b32 v68, v52
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v67, v51
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v81, v14 :: v_dual_mov_b32 v80, v13
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v79, v12 :: v_dual_mov_b32 v78, v11
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v77, v10 :: v_dual_mov_b32 v76, v9
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v75, v8
+; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[25:32], v[0:7], v[0:7], v[25:32]
+; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[42:49], v[17:24], v[17:24], v[42:49]
+; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[59:66], v[34:41], v[34:41], v[59:66]
+; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[67:74], v[51:58], v[51:58], v[67:74]
+; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[75:82], v[8:15], v[8:15], v[75:82]
+; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[29:32] offset:16
+; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[25:28]
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[46:49] offset:2064
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[42:45] offset:2048
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[63:66] offset:4112
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[59:62] offset:4096
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[71:74] offset:6160
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[67:70] offset:6144
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[79:82] offset:8208
+; EXACTCUTOFF-NEXT: ds_store_b128 v33, v[75:78] offset:8192
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(10) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
index 0a1df4267e619..00a6bb4dd136c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
@@ -95,16 +95,16 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_tied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_tied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_dual_mov_b32 v51, v39 :: v_dual_mov_b32 v50, v38
-; W32-NEXT: v_dual_mov_b32 v49, v37 :: v_dual_mov_b32 v48, v36
-; W32-NEXT: v_dual_mov_b32 v47, v35 :: v_dual_mov_b32 v46, v34
-; W32-NEXT: v_dual_mov_b32 v45, v33 :: v_dual_mov_b32 v44, v32
+; W32-NEXT: v_dual_mov_b32 v58, v39 :: v_dual_mov_b32 v57, v38
+; W32-NEXT: v_dual_mov_b32 v56, v37 :: v_dual_mov_b32 v55, v36
+; W32-NEXT: v_dual_mov_b32 v54, v35 :: v_dual_mov_b32 v53, v34
+; W32-NEXT: v_dual_mov_b32 v52, v33 :: v_dual_mov_b32 v51, v32
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[44:51]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[51:58]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
@@ -170,16 +170,16 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_tied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_tied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_dual_mov_b32 v51, v39 :: v_dual_mov_b32 v50, v38
-; W32-NEXT: v_dual_mov_b32 v49, v37 :: v_dual_mov_b32 v48, v36
-; W32-NEXT: v_dual_mov_b32 v47, v35 :: v_dual_mov_b32 v46, v34
-; W32-NEXT: v_dual_mov_b32 v45, v33 :: v_dual_mov_b32 v44, v32
+; W32-NEXT: v_dual_mov_b32 v58, v39 :: v_dual_mov_b32 v57, v38
+; W32-NEXT: v_dual_mov_b32 v56, v37 :: v_dual_mov_b32 v55, v36
+; W32-NEXT: v_dual_mov_b32 v54, v35 :: v_dual_mov_b32 v53, v34
+; W32-NEXT: v_dual_mov_b32 v52, v33 :: v_dual_mov_b32 v51, v32
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[44:51]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[51:58]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
new file mode 100644
index 0000000000000..de2922e811caf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
@@ -0,0 +1,610 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GCN,GFX1100 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 < %s | FileCheck -check-prefix=GCN,GFX1151 %s
+
+; Bank-conflict avoidance hint for v_wmma_* operands: the allocator should keep
+; the WMMA source operands (and the accumulator/result) in distinct VGPR banks
+; (bank = first VGPR % 4) without inflating the VGPR footprint on low-pressure
+; functions. See issue #204254.
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workgroup.id.x()
+declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half>, <16 x half>, <8 x float>)
+
+; A single WMMA in an ordinary (non-kernel) function. This must not blow up the
+; VGPR footprint: separating banks here would require climbing above the
+; argument registers, which costs more occupancy than it saves, so the hint
+; should stay opportunistic and keep the footprint near baseline.
+define void @wmma_f32_single(ptr %a, ptr %b, ptr %c, ptr %out) {
+; GCN-LABEL: wmma_f32_single:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: flat_load_b128 v[13:16], v[0:1] offset:16
+; GCN-NEXT: flat_load_b128 v[9:12], v[0:1]
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: flat_load_b128 v[22:25], v[2:3] offset:16
+; GCN-NEXT: flat_load_b128 v[18:21], v[2:3]
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: flat_load_b128 v[31:34], v[4:5] offset:16
+; GCN-NEXT: flat_load_b128 v[27:30], v[4:5]
+; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[27:34], v[9:16], v[18:25], v[27:34]
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: flat_store_b128 v[6:7], v[31:34] offset:16
+; GCN-NEXT: flat_store_b128 v[6:7], v[27:30]
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %loada = load <16 x half>, ptr %a
+ %loadb = load <16 x half>, ptr %b
+ %loadc = load <8 x float>, ptr %c
+ %wmma = tail call contract <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %loada, <16 x half> %loadb, <8 x float> %loadc)
+ store <8 x float> %wmma, ptr %out
+ ret void
+}
+
+; A WMMA accumulated across a loop in a kernel. The A/B operands and the
+; accumulator should land in separate banks.
+define protected amdgpu_kernel void @wmma_f32_accumulate_loop(ptr addrspace(1) noundef writeonly captures(none) %0, ptr addrspace(1) noundef readonly captures(none) %1, ptr addrspace(1) noundef readonly captures(none) %2, i32 noundef %3) local_unnamed_addr {
+; GFX1100-LABEL: wmma_f32_accumulate_loop:
+; GFX1100: ; %bb.0:
+; GFX1100-NEXT: s_clause 0x2
+; GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x18
+; GFX1100-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1100-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
+; GFX1100-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1100-NEXT: s_mov_b32 s6, s13
+; GFX1100-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-NEXT: s_cmp_lt_i32 s7, 1
+; GFX1100-NEXT: s_cbranch_scc1 .LBB1_3
+; GFX1100-NEXT: ; %bb.1: ; %.preheader
+; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 5, v8
+; GFX1100-NEXT: s_clause 0x3
+; GFX1100-NEXT: global_load_b128 v[13:16], v0, s[2:3] offset:16
+; GFX1100-NEXT: global_load_b128 v[9:12], v0, s[2:3]
+; GFX1100-NEXT: global_load_b128 v[22:25], v0, s[4:5] offset:16
+; GFX1100-NEXT: global_load_b128 v[18:21], v0, s[4:5]
+; GFX1100-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT: v_mov_b32_e32 v1, v0
+; GFX1100-NEXT: v_mov_b32_e32 v2, v0
+; GFX1100-NEXT: v_mov_b32_e32 v3, v0
+; GFX1100-NEXT: v_mov_b32_e32 v4, v0
+; GFX1100-NEXT: v_mov_b32_e32 v5, v0
+; GFX1100-NEXT: v_mov_b32_e32 v6, v0
+; GFX1100-NEXT: v_mov_b32_e32 v7, v0
+; GFX1100-NEXT: .LBB1_2: ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT: s_waitcnt vmcnt(0)
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1100-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[9:16], v[18:25], v[0:7]
+; GFX1100-NEXT: s_add_i32 s7, s7, -1
+; GFX1100-NEXT: s_cmp_eq_u32 s7, 0
+; GFX1100-NEXT: s_cbranch_scc0 .LBB1_2
+; GFX1100-NEXT: s_branch .LBB1_4
+; GFX1100-NEXT: .LBB1_3:
+; GFX1100-NEXT: v_mov_b32_e32 v0, 0
+; GFX1100-NEXT: .LBB1_4: ; %Flow26
+; GFX1100-NEXT: s_mov_b32 s7, 0
+; GFX1100-NEXT: s_mov_b32 s2, exec_lo
+; GFX1100-NEXT: v_cmpx_eq_u32_e32 0, v8
+; GFX1100-NEXT: s_cbranch_execz .LBB1_6
+; GFX1100-NEXT: ; %bb.5:
+; GFX1100-NEXT: s_lshl_b64 s[2:3], s[6:7], 2
+; GFX1100-NEXT: v_mov_b32_e32 v1, 0
+; GFX1100-NEXT: s_add_u32 s0, s0, s2
+; GFX1100-NEXT: s_addc_u32 s1, s1, s3
+; GFX1100-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1100-NEXT: .LBB1_6:
+; GFX1100-NEXT: s_endpgm
+;
+; GFX1151-LABEL: wmma_f32_accumulate_loop:
+; GFX1151: ; %bb.0:
+; GFX1151-NEXT: s_clause 0x2
+; GFX1151-NEXT: s_load_b32 s6, s[4:5], 0x18
+; GFX1151-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1151-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
+; GFX1151-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1151-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1151-NEXT: s_cmp_lt_i32 s6, 1
+; GFX1151-NEXT: s_cbranch_scc1 .LBB1_3
+; GFX1151-NEXT: ; %bb.1: ; %.preheader
+; GFX1151-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1151-NEXT: v_lshlrev_b32_e32 v0, 5, v8
+; GFX1151-NEXT: s_clause 0x3
+; GFX1151-NEXT: global_load_b128 v[13:16], v0, s[2:3] offset:16
+; GFX1151-NEXT: global_load_b128 v[9:12], v0, s[2:3]
+; GFX1151-NEXT: global_load_b128 v[22:25], v0, s[4:5] offset:16
+; GFX1151-NEXT: global_load_b128 v[18:21], v0, s[4:5]
+; GFX1151-NEXT: v_mov_b32_e32 v0, 0
+; GFX1151-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1151-NEXT: v_mov_b32_e32 v1, v0
+; GFX1151-NEXT: v_mov_b32_e32 v2, v0
+; GFX1151-NEXT: v_mov_b32_e32 v3, v0
+; GFX1151-NEXT: v_mov_b32_e32 v4, v0
+; GFX1151-NEXT: v_mov_b32_e32 v5, v0
+; GFX1151-NEXT: v_mov_b32_e32 v6, v0
+; GFX1151-NEXT: v_mov_b32_e32 v7, v0
+; GFX1151-NEXT: .LBB1_2: ; =>This Inner Loop Header: Depth=1
+; GFX1151-NEXT: s_waitcnt vmcnt(0)
+; GFX1151-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1151-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[9:16], v[18:25], v[0:7]
+; GFX1151-NEXT: s_add_i32 s6, s6, -1
+; GFX1151-NEXT: s_cmp_eq_u32 s6, 0
+; GFX1151-NEXT: s_cbranch_scc0 .LBB1_2
+; GFX1151-NEXT: s_branch .LBB1_4
+; GFX1151-NEXT: .LBB1_3:
+; GFX1151-NEXT: v_mov_b32_e32 v0, 0
+; GFX1151-NEXT: .LBB1_4: ; %Flow26
+; GFX1151-NEXT: s_mov_b32 s9, 0
+; GFX1151-NEXT: s_mov_b32 s2, exec_lo
+; GFX1151-NEXT: v_cmpx_eq_u32_e32 0, v8
+; GFX1151-NEXT: s_cbranch_execz .LBB1_6
+; GFX1151-NEXT: ; %bb.5:
+; GFX1151-NEXT: s_lshl_b64 s[2:3], s[8:9], 2
+; GFX1151-NEXT: v_mov_b32_e32 v1, 0
+; GFX1151-NEXT: s_add_u32 s0, s0, s2
+; GFX1151-NEXT: s_addc_u32 s1, s1, s3
+; GFX1151-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1151-NEXT: .LBB1_6:
+; GFX1151-NEXT: s_endpgm
+ %5 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %6 = zext nneg i32 %5 to i64
+ %7 = getelementptr inbounds nuw <16 x half>, ptr addrspace(1) %1, i64 %6
+ %8 = load <16 x half>, ptr addrspace(1) %7, align 32
+ %9 = getelementptr inbounds nuw <16 x half>, ptr addrspace(1) %2, i64 %6
+ %10 = load <16 x half>, ptr addrspace(1) %9, align 32
+ %11 = icmp sgt i32 %3, 0
+ br i1 %11, label %15, label %12
+
+12: ; preds = %15, %4
+ %13 = phi <8 x float> [ zeroinitializer, %4 ], [ %18, %15 ]
+ %14 = icmp eq i32 %5, 0
+ br i1 %14, label %21, label %26
+
+15: ; preds = %4, %15
+ %16 = phi i32 [ %19, %15 ], [ 0, %4 ]
+ %17 = phi <8 x float> [ %18, %15 ], [ zeroinitializer, %4 ]
+ %18 = tail call contract <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %8, <16 x half> %10, <8 x float> %17)
+ %19 = add nuw nsw i32 %16, 1
+ %20 = icmp eq i32 %19, %3
+ br i1 %20, label %12, label %15
+
+21: ; preds = %12
+ %22 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %23 = zext i32 %22 to i64
+ %24 = getelementptr inbounds nuw float, ptr addrspace(1) %0, i64 %23
+ %25 = extractelement <8 x float> %13, i64 0
+ store float %25, ptr addrspace(1) %24, align 4
+ br label %26
+
+26: ; preds = %21, %12
+ ret void
+}
+
+; Three independent WMMAs under maximum register pressure
+; ("amdgpu-waves-per-eu"="256,256" clamps to the smallest VGPR budget). Even
+; with no slack, all three should get their A/B/C operands in distinct banks.
+define amdgpu_kernel void @wmma_f32_3x_high_pressure(ptr addrspace(1) %pa, ptr addrspace(1) %pb, ptr addrspace(1) %out) #1 {
+; GCN-LABEL: wmma_f32_3x_high_pressure:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: s_load_b128 s[44:47], s[4:5], 0x0
+; GCN-NEXT: s_load_b64 s[34:35], s[4:5], 0x10
+; GCN-NEXT: s_mov_b32 s52, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_load_b256 s[0:7], s[44:45], 0x0
+; GCN-NEXT: s_load_b256 s[8:15], s[46:47], 0x0
+; GCN-NEXT: s_load_b256 s[16:23], s[44:45], 0x20
+; GCN-NEXT: s_load_b256 s[24:31], s[46:47], 0x20
+; GCN-NEXT: s_load_b256 s[36:43], s[44:45], 0x40
+; GCN-NEXT: s_load_b256 s[44:51], s[46:47], 0x40
+; GCN-NEXT: s_mov_b32 s53, s52
+; GCN-NEXT: s_mov_b32 s54, s52
+; GCN-NEXT: s_mov_b32 s55, s52
+; GCN-NEXT: s_mov_b32 s56, s52
+; GCN-NEXT: s_mov_b32 s57, s52
+; GCN-NEXT: s_mov_b32 s58, s52
+; GCN-NEXT: s_mov_b32 s59, s52
+; GCN-NEXT: v_dual_mov_b32 v0, s52 :: v_dual_mov_b32 v3, s55
+; GCN-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s53
+; GCN-NEXT: v_dual_mov_b32 v2, s54 :: v_dual_mov_b32 v5, s57
+; GCN-NEXT: v_dual_mov_b32 v4, s56 :: v_dual_mov_b32 v7, s59
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v21, s4
+; GCN-NEXT: v_dual_mov_b32 v18, s1 :: v_dual_mov_b32 v41, s15
+; GCN-NEXT: v_dual_mov_b32 v6, s58 :: v_dual_mov_b32 v23, s6
+; GCN-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v19, s2
+; GCN-NEXT: v_dual_mov_b32 v20, s3 :: v_dual_mov_b32 v17, s0
+; GCN-NEXT: v_dual_mov_b32 v40, s14 :: v_dual_mov_b32 v39, s13
+; GCN-NEXT: v_dual_mov_b32 v38, s12 :: v_dual_mov_b32 v37, s11
+; GCN-NEXT: v_dual_mov_b32 v36, s10 :: v_dual_mov_b32 v35, s9
+; GCN-NEXT: v_mov_b32_e32 v34, s8
+; GCN-NEXT: v_dual_mov_b32 v58, s23 :: v_dual_mov_b32 v55, s20
+; GCN-NEXT: v_dual_mov_b32 v32, s31 :: v_dual_mov_b32 v29, s28
+; GCN-NEXT: v_dual_mov_b32 v57, s22 :: v_dual_mov_b32 v56, s21
+; GCN-NEXT: v_dual_mov_b32 v53, s18 :: v_dual_mov_b32 v54, s19
+; GCN-NEXT: v_dual_mov_b32 v51, s16 :: v_dual_mov_b32 v52, s17
+; GCN-NEXT: v_dual_mov_b32 v31, s30 :: v_dual_mov_b32 v30, s29
+; GCN-NEXT: v_dual_mov_b32 v27, s26 :: v_dual_mov_b32 v28, s27
+; GCN-NEXT: v_dual_mov_b32 v25, s24 :: v_dual_mov_b32 v26, s25
+; GCN-NEXT: v_dual_mov_b32 v49, s43 :: v_dual_mov_b32 v66, s51
+; GCN-NEXT: v_dual_mov_b32 v63, s48 :: v_dual_mov_b32 v48, s42
+; GCN-NEXT: v_dual_mov_b32 v47, s41 :: v_dual_mov_b32 v46, s40
+; GCN-NEXT: v_dual_mov_b32 v45, s39 :: v_dual_mov_b32 v44, s38
+; GCN-NEXT: v_dual_mov_b32 v43, s37 :: v_dual_mov_b32 v42, s36
+; GCN-NEXT: v_dual_mov_b32 v65, s50 :: v_dual_mov_b32 v64, s49
+; GCN-NEXT: v_dual_mov_b32 v61, s46 :: v_dual_mov_b32 v62, s47
+; GCN-NEXT: v_dual_mov_b32 v59, s44 :: v_dual_mov_b32 v60, s45
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[17:24], v[34:41], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[34:41], v[51:58], v[25:32], v[0:7]
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[42:49], v[59:66], v[0:7]
+; GCN-NEXT: s_clause 0x5
+; GCN-NEXT: global_store_b128 v16, v[12:15], s[34:35] offset:16
+; GCN-NEXT: global_store_b128 v16, v[8:11], s[34:35]
+; GCN-NEXT: global_store_b128 v16, v[38:41], s[34:35] offset:48
+; GCN-NEXT: global_store_b128 v16, v[34:37], s[34:35] offset:32
+; GCN-NEXT: global_store_b128 v16, v[4:7], s[34:35] offset:80
+; GCN-NEXT: global_store_b128 v16, v[0:3], s[34:35] offset:64
+; GCN-NEXT: s_endpgm
+ %pa0 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 0
+ %a0 = load <16 x half>, ptr addrspace(1) %pa0
+ %pb0 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 0
+ %b0 = load <16 x half>, ptr addrspace(1) %pb0
+ %c0 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a0, <16 x half> %b0, <8 x float> zeroinitializer)
+ %pa1 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 1
+ %a1 = load <16 x half>, ptr addrspace(1) %pa1
+ %pb1 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 1
+ %b1 = load <16 x half>, ptr addrspace(1) %pb1
+ %c1 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a1, <16 x half> %b1, <8 x float> zeroinitializer)
+ %pa2 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 2
+ %a2 = load <16 x half>, ptr addrspace(1) %pa2
+ %pb2 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 2
+ %b2 = load <16 x half>, ptr addrspace(1) %pb2
+ %c2 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a2, <16 x half> %b2, <8 x float> zeroinitializer)
+ %po0 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 0
+ store <8 x float> %c0, ptr addrspace(1) %po0
+ %po1 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 1
+ store <8 x float> %c1, ptr addrspace(1) %po1
+ %po2 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 2
+ store <8 x float> %c2, ptr addrspace(1) %po2
+ ret void
+}
+
+; Twelve independent WMMAs near the VGPR budget ("amdgpu-waves-per-eu"="10,10").
+; This is the high-pressure stress case: the hint should separate banks on as
+; many WMMAs as the per-bank region capacity allows without introducing spills.
+define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrspace(1) %pb, ptr addrspace(1) %out) #2 {
+; GCN-LABEL: wmma_f32_12x_pressure:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GCN-NEXT: ; implicit-def: $vgpr65 : SGPR spill to VGPR lane
+; GCN-NEXT: s_mov_b32 s8, 0
+; GCN-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
+; GCN-NEXT: s_mov_b32 s14, s8
+; GCN-NEXT: s_mov_b32 s9, s8
+; GCN-NEXT: s_mov_b32 s10, s8
+; GCN-NEXT: s_mov_b32 s11, s8
+; GCN-NEXT: s_mov_b32 s12, s8
+; GCN-NEXT: s_mov_b32 s13, s8
+; GCN-NEXT: s_mov_b32 s15, s8
+; GCN-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
+; GCN-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v1, s9
+; GCN-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
+; GCN-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_load_b256 s[68:75], s[0:1], 0x60
+; GCN-NEXT: s_load_b256 s[76:83], s[2:3], 0x60
+; GCN-NEXT: s_load_b256 s[84:91], s[0:1], 0x80
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: s_load_b256 s[92:99], s[2:3], 0x80
+; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0x20
+; GCN-NEXT: s_load_b256 s[24:31], s[0:1], 0x0
+; GCN-NEXT: s_load_b256 s[44:51], s[2:3], 0x0
+; GCN-NEXT: v_mov_b32_e32 v6, s14
+; GCN-NEXT: s_clause 0x1
+; GCN-NEXT: s_load_b256 s[36:43], s[0:1], 0x20
+; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x40
+; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x40
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_dual_mov_b32 v49, s68 :: v_dual_mov_b32 v56, s75
+; GCN-NEXT: v_dual_mov_b32 v89, s83 :: v_dual_mov_b32 v122, s91
+; GCN-NEXT: v_mov_b32_e32 v119, s88
+; GCN-NEXT: v_writelane_b32 v65, s16, 0
+; GCN-NEXT: v_dual_mov_b32 v40, s31 :: v_dual_mov_b32 v37, s28
+; GCN-NEXT: v_dual_mov_b32 v39, s30 :: v_dual_mov_b32 v38, s29
+; GCN-NEXT: v_mov_b32_e32 v35, s26
+; GCN-NEXT: v_writelane_b32 v65, s17, 1
+; GCN-NEXT: v_dual_mov_b32 v36, s27 :: v_dual_mov_b32 v33, s24
+; GCN-NEXT: v_dual_mov_b32 v34, s25 :: v_dual_mov_b32 v73, s51
+; GCN-NEXT: v_writelane_b32 v65, s18, 2
+; GCN-NEXT: s_load_b256 s[24:31], s[0:1], 0xc0
+; GCN-NEXT: v_dual_mov_b32 v106, s43 :: v_dual_mov_b32 v103, s40
+; GCN-NEXT: v_dual_mov_b32 v66, s44 :: v_dual_mov_b32 v105, s42
+; GCN-NEXT: v_writelane_b32 v65, s19, 3
+; GCN-NEXT: v_dual_mov_b32 v104, s41 :: v_dual_mov_b32 v101, s38
+; GCN-NEXT: v_dual_mov_b32 v102, s39 :: v_dual_mov_b32 v99, s36
+; GCN-NEXT: v_writelane_b32 v65, s20, 4
+; GCN-NEXT: v_dual_mov_b32 v114, s67 :: v_dual_mov_b32 v111, s64
+; GCN-NEXT: v_dual_mov_b32 v72, s50 :: v_dual_mov_b32 v71, s49
+; GCN-NEXT: v_writelane_b32 v65, s21, 5
+; GCN-NEXT: v_dual_mov_b32 v70, s48 :: v_dual_mov_b32 v69, s47
+; GCN-NEXT: v_dual_mov_b32 v68, s46 :: v_dual_mov_b32 v67, s45
+; GCN-NEXT: v_writelane_b32 v65, s22, 6
+; GCN-NEXT: v_dual_mov_b32 v112, s65 :: v_dual_mov_b32 v109, s62
+; GCN-NEXT: v_dual_mov_b32 v110, s63 :: v_dual_mov_b32 v107, s60
+; GCN-NEXT: v_writelane_b32 v65, s23, 7
+; GCN-NEXT: s_load_b256 s[8:15], s[0:1], 0xa0
+; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0xa0
+; GCN-NEXT: v_mov_b32_e32 v108, s61
+; GCN-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v57, s92
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_writelane_b32 v65, s24, 8
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[123:130], v[33:40], v[66:73], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v50, s69 :: v_dual_mov_b32 v51, s70
+; GCN-NEXT: v_writelane_b32 v65, s25, 9
+; GCN-NEXT: v_dual_mov_b32 v52, s71 :: v_dual_mov_b32 v53, s72
+; GCN-NEXT: v_dual_mov_b32 v54, s73 :: v_dual_mov_b32 v55, s74
+; GCN-NEXT: v_writelane_b32 v65, s26, 10
+; GCN-NEXT: v_dual_mov_b32 v88, s82 :: v_dual_mov_b32 v87, s81
+; GCN-NEXT: v_dual_mov_b32 v86, s80 :: v_dual_mov_b32 v85, s79
+; GCN-NEXT: v_writelane_b32 v65, s27, 11
+; GCN-NEXT: v_dual_mov_b32 v84, s78 :: v_dual_mov_b32 v83, s77
+; GCN-NEXT: v_dual_mov_b32 v82, s76 :: v_dual_mov_b32 v121, s90
+; GCN-NEXT: v_writelane_b32 v65, s28, 12
+; GCN-NEXT: v_mov_b32_e32 v64, s99
+; GCN-NEXT: v_dual_mov_b32 v120, s89 :: v_dual_mov_b32 v117, s86
+; GCN-NEXT: v_dual_mov_b32 v118, s87 :: v_dual_mov_b32 v115, s84
+; GCN-NEXT: v_writelane_b32 v65, s29, 13
+; GCN-NEXT: v_dual_mov_b32 v58, s93 :: v_dual_mov_b32 v59, s94
+; GCN-NEXT: v_dual_mov_b32 v60, s95 :: v_dual_mov_b32 v61, s96
+; GCN-NEXT: v_writelane_b32 v65, s30, 14
+; GCN-NEXT: v_dual_mov_b32 v62, s97 :: v_dual_mov_b32 v63, s98
+; GCN-NEXT: v_dual_mov_b32 v81, s59 :: v_dual_mov_b32 v80, s58
+; GCN-NEXT: v_mov_b32_e32 v79, s57
+; GCN-NEXT: v_writelane_b32 v65, s31, 15
+; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xc0
+; GCN-NEXT: v_dual_mov_b32 v78, s56 :: v_dual_mov_b32 v77, s55
+; GCN-NEXT: v_dual_mov_b32 v76, s54 :: v_dual_mov_b32 v75, s53
+; GCN-NEXT: v_dual_mov_b32 v74, s52 :: v_dual_mov_b32 v113, s66
+; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x100
+; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x100
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[33:40], v[74:81], v[107:114], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v114, s23 :: v_dual_mov_b32 v111, s20
+; GCN-NEXT: v_dual_mov_b32 v113, s22 :: v_dual_mov_b32 v112, s21
+; GCN-NEXT: v_dual_mov_b32 v109, s18 :: v_dual_mov_b32 v110, s19
+; GCN-NEXT: v_mov_b32_e32 v107, s16
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_writelane_b32 v65, s24, 16
+; GCN-NEXT: v_writelane_b32 v65, s25, 17
+; GCN-NEXT: v_dual_mov_b32 v97, s67 :: v_dual_mov_b32 v96, s66
+; GCN-NEXT: v_mov_b32_e32 v95, s65
+; GCN-NEXT: v_writelane_b32 v65, s26, 18
+; GCN-NEXT: v_dual_mov_b32 v94, s64 :: v_dual_mov_b32 v93, s63
+; GCN-NEXT: v_dual_mov_b32 v92, s62 :: v_dual_mov_b32 v91, s61
+; GCN-NEXT: v_writelane_b32 v65, s27, 19
+; GCN-NEXT: v_writelane_b32 v65, s28, 20
+; GCN-NEXT: v_writelane_b32 v65, s29, 21
+; GCN-NEXT: v_writelane_b32 v65, s30, 22
+; GCN-NEXT: v_writelane_b32 v65, s31, 23
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT: v_readlane_b32 s24, v65, 0
+; GCN-NEXT: v_readlane_b32 s31, v65, 7
+; GCN-NEXT: v_readlane_b32 s25, v65, 1
+; GCN-NEXT: v_readlane_b32 s26, v65, 2
+; GCN-NEXT: v_readlane_b32 s27, v65, 3
+; GCN-NEXT: v_readlane_b32 s28, v65, 4
+; GCN-NEXT: v_readlane_b32 s29, v65, 5
+; GCN-NEXT: v_readlane_b32 s30, v65, 6
+; GCN-NEXT: v_mov_b32_e32 v48, s31
+; GCN-NEXT: v_mov_b32_e32 v100, s37
+; GCN-NEXT: s_load_b256 s[36:43], s[0:1], 0xe0
+; GCN-NEXT: v_dual_mov_b32 v46, s29 :: v_dual_mov_b32 v43, s26
+; GCN-NEXT: v_mov_b32_e32 v47, s30
+; GCN-NEXT: v_dual_mov_b32 v45, s28 :: v_dual_mov_b32 v44, s27
+; GCN-NEXT: v_dual_mov_b32 v41, s24 :: v_dual_mov_b32 v42, s25
+; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xe0
+; GCN-NEXT: v_readlane_b32 s44, v65, 8
+; GCN-NEXT: v_readlane_b32 s45, v65, 9
+; GCN-NEXT: v_readlane_b32 s46, v65, 10
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[66:73], v[99:106], v[41:48], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v48, s15 :: v_dual_mov_b32 v45, s12
+; GCN-NEXT: v_dual_mov_b32 v47, s14 :: v_dual_mov_b32 v46, s13
+; GCN-NEXT: v_dual_mov_b32 v43, s10 :: v_dual_mov_b32 v44, s11
+; GCN-NEXT: v_dual_mov_b32 v41, s8 :: v_dual_mov_b32 v42, s9
+; GCN-NEXT: v_readlane_b32 s47, v65, 11
+; GCN-NEXT: v_readlane_b32 s48, v65, 12
+; GCN-NEXT: v_readlane_b32 s49, v65, 13
+; GCN-NEXT: v_readlane_b32 s50, v65, 14
+; GCN-NEXT: v_readlane_b32 s51, v65, 15
+; GCN-NEXT: v_mov_b32_e32 v108, s17
+; GCN-NEXT: s_load_b256 s[8:15], s[0:1], 0x120
+; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0x120
+; GCN-NEXT: s_load_b256 s[68:75], s[0:1], 0x140
+; GCN-NEXT: s_load_b256 s[76:83], s[2:3], 0x140
+; GCN-NEXT: s_load_b256 s[84:91], s[0:1], 0x160
+; GCN-NEXT: s_load_b256 s[92:99], s[2:3], 0x160
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[99:106], v[49:56], v[82:89], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[74:81], v[115:122], v[57:64], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[82:89], v[41:48], v[107:114], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v41, s44 :: v_dual_mov_b32 v42, s45
+; GCN-NEXT: v_dual_mov_b32 v43, s46 :: v_dual_mov_b32 v44, s47
+; GCN-NEXT: v_dual_mov_b32 v45, s48 :: v_dual_mov_b32 v46, s49
+; GCN-NEXT: v_dual_mov_b32 v47, s50 :: v_dual_mov_b32 v48, s51
+; GCN-NEXT: v_readlane_b32 s44, v65, 16
+; GCN-NEXT: v_readlane_b32 s51, v65, 23
+; GCN-NEXT: v_readlane_b32 s45, v65, 17
+; GCN-NEXT: v_readlane_b32 s46, v65, 18
+; GCN-NEXT: v_readlane_b32 s47, v65, 19
+; GCN-NEXT: v_readlane_b32 s48, v65, 20
+; GCN-NEXT: v_readlane_b32 s49, v65, 21
+; GCN-NEXT: v_readlane_b32 s50, v65, 22
+; GCN-NEXT: v_mov_b32_e32 v114, s51
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_dual_mov_b32 v56, s43 :: v_dual_mov_b32 v53, s40
+; GCN-NEXT: v_dual_mov_b32 v122, s31 :: v_dual_mov_b32 v119, s28
+; GCN-NEXT: v_dual_mov_b32 v108, s45 :: v_dual_mov_b32 v55, s42
+; GCN-NEXT: v_dual_mov_b32 v54, s41 :: v_dual_mov_b32 v51, s38
+; GCN-NEXT: v_dual_mov_b32 v52, s39 :: v_dual_mov_b32 v49, s36
+; GCN-NEXT: v_dual_mov_b32 v50, s37 :: v_dual_mov_b32 v121, s30
+; GCN-NEXT: v_dual_mov_b32 v120, s29 :: v_dual_mov_b32 v117, s26
+; GCN-NEXT: v_dual_mov_b32 v118, s27 :: v_dual_mov_b32 v115, s24
+; GCN-NEXT: v_mov_b32_e32 v116, s25
+; GCN-NEXT: v_dual_mov_b32 v64, s59 :: v_dual_mov_b32 v61, s56
+; GCN-NEXT: v_dual_mov_b32 v113, s50 :: v_dual_mov_b32 v112, s49
+; GCN-NEXT: v_mov_b32_e32 v109, s46
+; GCN-NEXT: v_dual_mov_b32 v111, s48 :: v_dual_mov_b32 v110, s47
+; GCN-NEXT: v_dual_mov_b32 v107, s44 :: v_dual_mov_b32 v58, s53
+; GCN-NEXT: v_dual_mov_b32 v63, s58 :: v_dual_mov_b32 v62, s57
+; GCN-NEXT: v_dual_mov_b32 v59, s54 :: v_dual_mov_b32 v60, s55
+; GCN-NEXT: v_dual_mov_b32 v57, s52 :: v_dual_mov_b32 v90, s60
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[16:23], v[49:56], v[115:122], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v122, s23 :: v_dual_mov_b32 v119, s20
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[41:48], v[107:114], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v48, s15 :: v_dual_mov_b32 v45, s12
+; GCN-NEXT: v_dual_mov_b32 v116, s17 :: v_dual_mov_b32 v49, s68
+; GCN-NEXT: v_mov_b32_e32 v47, s14
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[107:114], v[57:64], v[90:97], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v56, s75 :: v_dual_mov_b32 v97, s83
+; GCN-NEXT: v_dual_mov_b32 v90, s76 :: v_dual_mov_b32 v57, s84
+; GCN-NEXT: v_dual_mov_b32 v24, s92 :: v_dual_mov_b32 v27, s95
+; GCN-NEXT: v_dual_mov_b32 v46, s13 :: v_dual_mov_b32 v43, s10
+; GCN-NEXT: v_dual_mov_b32 v44, s11 :: v_dual_mov_b32 v41, s8
+; GCN-NEXT: v_dual_mov_b32 v42, s9 :: v_dual_mov_b32 v121, s22
+; GCN-NEXT: v_dual_mov_b32 v120, s21 :: v_dual_mov_b32 v117, s18
+; GCN-NEXT: v_dual_mov_b32 v118, s19 :: v_dual_mov_b32 v115, s16
+; GCN-NEXT: v_dual_mov_b32 v50, s69 :: v_dual_mov_b32 v51, s70
+; GCN-NEXT: v_dual_mov_b32 v52, s71 :: v_dual_mov_b32 v53, s72
+; GCN-NEXT: v_dual_mov_b32 v54, s73 :: v_dual_mov_b32 v55, s74
+; GCN-NEXT: v_dual_mov_b32 v96, s82 :: v_dual_mov_b32 v95, s81
+; GCN-NEXT: v_dual_mov_b32 v94, s80 :: v_dual_mov_b32 v93, s79
+; GCN-NEXT: v_dual_mov_b32 v92, s78 :: v_dual_mov_b32 v91, s77
+; GCN-NEXT: v_dual_mov_b32 v58, s85 :: v_dual_mov_b32 v59, s86
+; GCN-NEXT: v_dual_mov_b32 v60, s87 :: v_dual_mov_b32 v61, s88
+; GCN-NEXT: v_dual_mov_b32 v62, s89 :: v_dual_mov_b32 v63, s90
+; GCN-NEXT: v_dual_mov_b32 v64, s91 :: v_dual_mov_b32 v25, s93
+; GCN-NEXT: v_dual_mov_b32 v26, s94 :: v_dual_mov_b32 v29, s97
+; GCN-NEXT: v_dual_mov_b32 v28, s96 :: v_dual_mov_b32 v31, s99
+; GCN-NEXT: v_mov_b32_e32 v30, s98
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[131:138], v[41:48], v[115:122], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[115:122], v[49:56], v[90:97], v[0:7]
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[57:64], v[24:31], v[0:7]
+; GCN-NEXT: s_clause 0x17
+; GCN-NEXT: global_store_b128 v32, v[127:130], s[4:5] offset:16
+; GCN-NEXT: global_store_b128 v32, v[123:126], s[4:5]
+; GCN-NEXT: global_store_b128 v32, v[70:73], s[4:5] offset:48
+; GCN-NEXT: global_store_b128 v32, v[66:69], s[4:5] offset:32
+; GCN-NEXT: global_store_b128 v32, v[37:40], s[4:5] offset:80
+; GCN-NEXT: global_store_b128 v32, v[33:36], s[4:5] offset:64
+; GCN-NEXT: global_store_b128 v32, v[103:106], s[4:5] offset:112
+; GCN-NEXT: global_store_b128 v32, v[99:102], s[4:5] offset:96
+; GCN-NEXT: global_store_b128 v32, v[78:81], s[4:5] offset:144
+; GCN-NEXT: global_store_b128 v32, v[74:77], s[4:5] offset:128
+; GCN-NEXT: global_store_b128 v32, v[86:89], s[4:5] offset:176
+; GCN-NEXT: global_store_b128 v32, v[82:85], s[4:5] offset:160
+; GCN-NEXT: global_store_b128 v32, v[12:15], s[4:5] offset:208
+; GCN-NEXT: global_store_b128 v32, v[8:11], s[4:5] offset:192
+; GCN-NEXT: global_store_b128 v32, v[20:23], s[4:5] offset:240
+; GCN-NEXT: global_store_b128 v32, v[16:19], s[4:5] offset:224
+; GCN-NEXT: global_store_b128 v32, v[111:114], s[4:5] offset:272
+; GCN-NEXT: global_store_b128 v32, v[107:110], s[4:5] offset:256
+; GCN-NEXT: global_store_b128 v32, v[135:138], s[4:5] offset:304
+; GCN-NEXT: global_store_b128 v32, v[131:134], s[4:5] offset:288
+; GCN-NEXT: global_store_b128 v32, v[119:122], s[4:5] offset:336
+; GCN-NEXT: global_store_b128 v32, v[115:118], s[4:5] offset:320
+; GCN-NEXT: global_store_b128 v32, v[4:7], s[4:5] offset:368
+; GCN-NEXT: global_store_b128 v32, v[0:3], s[4:5] offset:352
+; GCN-NEXT: s_nop 0
+; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT: s_endpgm
+ %pa0 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 0
+ %a0 = load <16 x half>, ptr addrspace(1) %pa0
+ %pb0 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 0
+ %b0 = load <16 x half>, ptr addrspace(1) %pb0
+ %c0 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a0, <16 x half> %b0, <8 x float> zeroinitializer)
+ %pa1 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 1
+ %a1 = load <16 x half>, ptr addrspace(1) %pa1
+ %pb1 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 1
+ %b1 = load <16 x half>, ptr addrspace(1) %pb1
+ %c1 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a1, <16 x half> %b1, <8 x float> zeroinitializer)
+ %pa2 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 2
+ %a2 = load <16 x half>, ptr addrspace(1) %pa2
+ %pb2 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 2
+ %b2 = load <16 x half>, ptr addrspace(1) %pb2
+ %c2 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a2, <16 x half> %b2, <8 x float> zeroinitializer)
+ %pa3 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 3
+ %a3 = load <16 x half>, ptr addrspace(1) %pa3
+ %pb3 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 3
+ %b3 = load <16 x half>, ptr addrspace(1) %pb3
+ %c3 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a3, <16 x half> %b3, <8 x float> zeroinitializer)
+ %pa4 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 4
+ %a4 = load <16 x half>, ptr addrspace(1) %pa4
+ %pb4 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 4
+ %b4 = load <16 x half>, ptr addrspace(1) %pb4
+ %c4 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a4, <16 x half> %b4, <8 x float> zeroinitializer)
+ %pa5 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 5
+ %a5 = load <16 x half>, ptr addrspace(1) %pa5
+ %pb5 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 5
+ %b5 = load <16 x half>, ptr addrspace(1) %pb5
+ %c5 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a5, <16 x half> %b5, <8 x float> zeroinitializer)
+ %pa6 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 6
+ %a6 = load <16 x half>, ptr addrspace(1) %pa6
+ %pb6 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 6
+ %b6 = load <16 x half>, ptr addrspace(1) %pb6
+ %c6 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a6, <16 x half> %b6, <8 x float> zeroinitializer)
+ %pa7 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 7
+ %a7 = load <16 x half>, ptr addrspace(1) %pa7
+ %pb7 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 7
+ %b7 = load <16 x half>, ptr addrspace(1) %pb7
+ %c7 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a7, <16 x half> %b7, <8 x float> zeroinitializer)
+ %pa8 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 8
+ %a8 = load <16 x half>, ptr addrspace(1) %pa8
+ %pb8 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 8
+ %b8 = load <16 x half>, ptr addrspace(1) %pb8
+ %c8 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a8, <16 x half> %b8, <8 x float> zeroinitializer)
+ %pa9 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 9
+ %a9 = load <16 x half>, ptr addrspace(1) %pa9
+ %pb9 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 9
+ %b9 = load <16 x half>, ptr addrspace(1) %pb9
+ %c9 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a9, <16 x half> %b9, <8 x float> zeroinitializer)
+ %pa10 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 10
+ %a10 = load <16 x half>, ptr addrspace(1) %pa10
+ %pb10 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 10
+ %b10 = load <16 x half>, ptr addrspace(1) %pb10
+ %c10 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a10, <16 x half> %b10, <8 x float> zeroinitializer)
+ %pa11 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 11
+ %a11 = load <16 x half>, ptr addrspace(1) %pa11
+ %pb11 = getelementptr <16 x half>, ptr addrspace(1) %pb, i64 11
+ %b11 = load <16 x half>, ptr addrspace(1) %pb11
+ %c11 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16(<16 x half> %a11, <16 x half> %b11, <8 x float> zeroinitializer)
+ %po0 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 0
+ store <8 x float> %c0, ptr addrspace(1) %po0
+ %po1 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 1
+ store <8 x float> %c1, ptr addrspace(1) %po1
+ %po2 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 2
+ store <8 x float> %c2, ptr addrspace(1) %po2
+ %po3 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 3
+ store <8 x float> %c3, ptr addrspace(1) %po3
+ %po4 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 4
+ store <8 x float> %c4, ptr addrspace(1) %po4
+ %po5 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 5
+ store <8 x float> %c5, ptr addrspace(1) %po5
+ %po6 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 6
+ store <8 x float> %c6, ptr addrspace(1) %po6
+ %po7 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 7
+ store <8 x float> %c7, ptr addrspace(1) %po7
+ %po8 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 8
+ store <8 x float> %c8, ptr addrspace(1) %po8
+ %po9 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 9
+ store <8 x float> %c9, ptr addrspace(1) %po9
+ %po10 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 10
+ store <8 x float> %c10, ptr addrspace(1) %po10
+ %po11 = getelementptr <8 x float>, ptr addrspace(1) %out, i64 11
+ store <8 x float> %c11, ptr addrspace(1) %po11
+ ret void
+}
+
+attributes #1 = { "amdgpu-waves-per-eu"="256,256" }
+attributes #2 = { "amdgpu-waves-per-eu"="10,10" }
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index 6c943da2f1dd7..7ff1236b40b42 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -409,20 +409,20 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x
; GFX1170-LABEL: test_wmma_f16_16x16x16_f16_negC_pack:
; GFX1170: ; %bb.0: ; %bb
; GFX1170-NEXT: s_clause 0x1
-; GFX1170-NEXT: flat_load_b128 v[12:15], v[8:9]
-; GFX1170-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16
+; GFX1170-NEXT: flat_load_b128 v[15:18], v[8:9]
+; GFX1170-NEXT: flat_load_b128 v[19:22], v[8:9] offset:16
; GFX1170-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
-; GFX1170-NEXT: v_mov_b16_e32 v14.h, v15.l
+; GFX1170-NEXT: v_mov_b16_e32 v17.h, v18.l
; GFX1170-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1170-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX1170-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX1170-NEXT: v_mov_b16_e32 v12.h, v13.l
+; GFX1170-NEXT: v_mov_b16_e32 v19.h, v20.l
+; GFX1170-NEXT: v_mov_b16_e32 v21.h, v22.l
+; GFX1170-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_dual_mov_b32 v13, v14 :: v_dual_mov_b32 v14, v16
-; GFX1170-NEXT: v_mov_b32_e32 v15, v18
+; GFX1170-NEXT: v_dual_mov_b32 v16, v17 :: v_dual_mov_b32 v17, v19
+; GFX1170-NEXT: v_mov_b32_e32 v18, v21
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1]
-; GFX1170-NEXT: global_store_b128 v[10:11], v[12:15], off
+; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[15:18], v[0:3], v[4:7], v[15:18] neg_lo:[0,0,1]
+; GFX1170-NEXT: global_store_b128 v[10:11], v[15:18], off
; GFX1170-NEXT: s_endpgm
bb:
%C = load <16 x half>, ptr %Caddr
>From 25985173c08b66b6f3701f10923fe033c72262d9 Mon Sep 17 00:00:00 2001
From: shore <372660931 at qq.com>
Date: Thu, 25 Jun 2026 11:52:10 +0800
Subject: [PATCH 2/6] fix comments
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 10 +-
.../Target/AMDGPU/GCNPreRAOptimizations.cpp | 67 +----
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 9 +
.../lib/Target/AMDGPU/SIMachineFunctionInfo.h | 15 +-
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 52 +++-
.../AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll | 12 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 276 +++++++++---------
.../CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll | 12 +-
llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll | 48 +--
llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll | 24 +-
10 files changed, 257 insertions(+), 268 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 705e76ebeb7e5..c01d7c7d7700f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1437,6 +1437,10 @@ defm VOP3PX2IncrementsVaVdstTwice : AMDGPUSubtargetFeature<"vop3px2-increments-v
"VOP3PX2 encoded instructions increment the VaVdst counter twice", GenPredicate=false
>;
+defm VGPRBankConflict: AMDGPUSubtargetFeature<"vgpr-bank-conflict",
+ "WMMA instruction will penalty for VGPR bank conflict"
+>;
+
//===----------------------------------------------------------------------===//
class GCNSubtargetFeatureGeneration <string Value,
@@ -1577,7 +1581,8 @@ def FeatureGFX11 : GCNSubtargetFeatureGeneration<"GFX11",
FeatureSadInsts, FeatureQsadInsts, FeatureMsadInsts, FeatureMqsadPkInsts,
FeatureMqsadInsts, FeatureCvtNormInsts,
FeatureCvtPkNormVOP2Insts, FeatureCvtPkNormVOP3Insts,
- FeatureInstCacheLineSize128, FeatureMTBUFInsts, FeatureFormattedMUBUFInsts
+ FeatureInstCacheLineSize128, FeatureMTBUFInsts, FeatureFormattedMUBUFInsts,
+ FeatureVGPRBankConflict
]
>;
@@ -1604,7 +1609,8 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
FeatureIEEEMinimumMaximumInsts, FeatureSALUMinimumMaximumInsts,
FeatureMinimum3Maximum3F32, FeatureMinimum3Maximum3F16,
FeatureAgentScopeFineGrainedRemoteMemoryAtomics, FeatureFlatOffsetBits24,
- FeatureFlatSignedOffset, FeatureInstCacheLineSize128
+ FeatureFlatSignedOffset, FeatureInstCacheLineSize128,
+ FeatureVGPRBankConflict
]
>;
diff --git a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
index d4c2530afcd26..825634d7af65b 100644
--- a/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNPreRAOptimizations.cpp
@@ -32,10 +32,8 @@
#include "GCNPreRAOptimizations.h"
#include "AMDGPU.h"
-#include "GCNRegPressure.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -56,8 +54,6 @@ class GCNPreRAOptimizationsImpl {
bool processReg(Register Reg);
void hintTrue16Copy(const MachineInstr &MI);
- void recordWMMABankSiblings(const MachineInstr &MI,
- SIMachineFunctionInfo &MFI);
bool optimizeBVHStack(MachineInstr &MI);
public:
@@ -266,41 +262,6 @@ void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
MRI->setRegAllocationHint(Dst, AMDGPURI::Size16, Src);
}
-void GCNPreRAOptimizationsImpl::recordWMMABankSiblings(
- const MachineInstr &MI, SIMachineFunctionInfo &MFI) {
- // The consecutive-allocation VGPR bank collision only happens when each
- // matrix operand is a multiple of 4 VGPRs (128 bits) wide; with narrower
- // operands the start registers already fall on different banks. Restrict the
- // hint to WMMAs whose matrix source operands are a multiple of 128 bits.
- auto Is128BitMultiple = [&](AMDGPU::OpName Name) {
- const MachineOperand *MO = TII->getNamedOperand(MI, Name);
- return MO && MO->isReg() && MO->getReg() &&
- TRI->getRegSizeInBits(MO->getReg(), *MRI) % 128 == 0;
- };
- if (!Is128BitMultiple(AMDGPU::OpName::src0) ||
- !Is128BitMultiple(AMDGPU::OpName::src1))
- return;
-
- // Record, for each virtual register operand, the other operands of this WMMA
- // so SIRegisterInfo::getRegAllocationHints can steer them onto distinct
- // VGPR banks. A register used by several WMMAs accumulates the union of their
- // siblings.
- SmallVector<Register, 4> Regs;
- for (const MachineOperand &MO : MI.operands())
- if (MO.isReg() && MO.getReg() && MO.getReg().isVirtual() &&
- !is_contained(Regs, MO.getReg()))
- Regs.push_back(MO.getReg());
-
- DenseMap<Register, SmallVector<Register, 3>> &Siblings =
- MFI.getWMMABankSiblings();
- for (Register Reg : Regs) {
- SmallVector<Register, 3> &S = Siblings[Reg];
- for (Register Other : Regs)
- if (Other != Reg && !is_contained(S, Other))
- S.push_back(Other);
- }
-}
-
bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
SmallVector<Register, 2> UseRegs;
@@ -360,15 +321,10 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
const bool HasBVHStack = ST.hasBVHDualAndBVH8Insts();
const bool HasRealTrue16 = ST.useRealTrue16Insts();
- // On gfx11/gfx12 the operands of a WMMA should be placed on distinct VGPR
- // banks to avoid an issue-latency penalty.
- const bool HasWMMABankHint = ST.hasFeature(AMDGPU::FeatureGFX11) ||
- ST.hasFeature(AMDGPU::FeatureGFX12);
- if (!HasRealTrue16 && !HasBVHStack && !HasWMMABankHint)
+ if (!HasRealTrue16 && !HasBVHStack)
return Changed;
- SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// Add RA hints to improve True16 COPY elimination.
@@ -376,11 +332,6 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
hintTrue16Copy(MI);
continue;
}
- // Record WMMA operand siblings for VGPR bank-conflict avoidance hints.
- if (HasWMMABankHint && SIInstrInfo::isWMMA(MI)) {
- recordWMMABankSiblings(MI, *MFI);
- continue;
- }
// Add implicit uses to avoid early wait on intersect ray instructions.
if (HasBVHStack &&
(MI.getOpcode() == AMDGPU::DS_BVH_STACK_RTN_B32 ||
@@ -392,21 +343,5 @@ bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
}
}
- // If any bank-sensitive WMMA was tagged, record the function's peak ArchVGPR
- // pressure (the actual VGPR demand). The bank hint anchors its reserved
- // regions to this instead of the top of the whole VGPR file.
- if (HasWMMABankHint && !MFI->getWMMABankSiblings().empty()) {
- unsigned Peak = 0;
- GCNUpwardRPTracker RPT(*LIS);
- for (const MachineBasicBlock &MBB : MF) {
- RPT.reset(*MRI, LIS->getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());
- for (const MachineInstr &MI : reverse(MBB)) {
- RPT.recede(MI);
- Peak = std::max(Peak, RPT.getPressure().getArchVGPRNum());
- }
- }
- MFI->setWMMAPeakVGPRPressure(Peak);
- }
-
return Changed;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index a4f854beaeebe..7865f682fd9fe 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1264,6 +1264,15 @@ void GCNScheduleDAGMILive::runSchedStages() {
Stage->finalizeGCNSchedStage();
}
+ // Record the function's peak ArchVGPR pressure from the per-region pressure
+ // we just computed, so later passes (e.g. the WMMA bank-conflict hint in
+ // SIRegisterInfo) can anchor to the real VGPR demand without constructing a
+ // standalone register-pressure tracker.
+ unsigned PeakVGPRPressure = 0;
+ for (const GCNRegPressure &RP : Pressure)
+ PeakVGPRPressure = std::max(PeakVGPRPressure, RP.getArchVGPRNum());
+ MFI.setPeakVGPRPressure(PeakVGPRPressure);
+
#ifdef DUMP_MAX_REG_PRESSURE
if (PrintMaxRPRegUsageAfterScheduler) {
dumpMaxRegPressure(MF, GCNRegPressure::VGPR, *LIS, MLI);
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 0717e48132aca..8d836ec175fa1 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -624,11 +624,12 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
// the hint hook does not rescan the function for every virtual register.
DenseMap<Register, SmallVector<Register, 3>> WMMABankSiblings;
- // Peak ArchVGPR pressure of the function (the "actual demand"), computed once
- // by GCNPreRAOptimizations. Used by the WMMA bank hint to anchor its reserved
- // bank regions to the top of the real footprint instead of the top of the
- // whole VGPR file, so it does not inflate the VGPR count of small kernels.
- unsigned WMMAPeakVGPRPressure = 0;
+ // Peak ArchVGPR pressure of the function (the "actual demand"), recorded by
+ // the machine scheduler (GCNScheduleDAGMILive) from the per-region pressure
+ // it already computes. Used by the WMMA bank hint to anchor its reserved bank
+ // regions to the top of the real footprint instead of the top of the whole
+ // VGPR file, so it does not inflate the VGPR count of small kernels.
+ unsigned PeakVGPRPressure = 0;
private:
Register VGPRForAGPRCopy;
@@ -672,8 +673,8 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
return WMMABankSiblings;
}
- unsigned getWMMAPeakVGPRPressure() const { return WMMAPeakVGPRPressure; }
- void setWMMAPeakVGPRPressure(unsigned P) { WMMAPeakVGPRPressure = P; }
+ unsigned getPeakVGPRPressure() const { return PeakVGPRPressure; }
+ void setPeakVGPRPressure(unsigned P) { PeakVGPRPressure = P; }
public:
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index ad18d7c940ddc..091ce000632a8 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4152,11 +4152,45 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
// Append v_wmma_* bank-conflict avoidance candidates (gfx11/gfx12). This only
// augments the candidate order for VirtReg and never changes BaseImplRetVal.
- addWMMABankConflictHints(VirtReg, Order, Hints, MF, VRM);
+ if (ST.hasVGPRBankConflict())
+ addWMMABankConflictHints(VirtReg, Order, Hints, MF, VRM);
return BaseImplRetVal;
}
+static void recordWMMABankSiblings(Register VirtReg, const MachineInstr &MI,
+ SmallVector<Register, 3> &Siblings,
+ const SIInstrInfo *TII,
+ const SIRegisterInfo *TRI,
+ const MachineRegisterInfo &MRI) {
+ // The consecutive-allocation VGPR bank collision only happens when each
+ // matrix operand is a multiple of 4 VGPRs (128 bits) wide; with narrower
+ // operands the start registers already fall on different banks. Restrict the
+ // hint to WMMAs whose matrix source operands are a multiple of 128 bits.
+ Register Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)->getReg();
+ Register Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)->getReg();
+ if (TRI->getRegSizeInBits(Src0, MRI) % 128 != 0 ||
+ TRI->getRegSizeInBits(Src1, MRI) % 128 != 0)
+ return;
+
+ Register Src2, Vdst;
+ const MachineOperand *MOSrc2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+ if (MOSrc2 && MOSrc2->isReg() && MOSrc2->getReg().isVirtual())
+ Src2 = MOSrc2->getReg();
+ const MachineOperand *MODst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
+ if (MODst && MODst->isReg() && MODst->getReg().isVirtual())
+ Vdst = MODst->getReg();
+
+ // Record, for each matrix operand, the other matrix operands of this WMMA so
+ // SIRegisterInfo::getRegAllocationHints can steer them onto distinct VGPR
+ // banks. A register used by several WMMAs accumulates the union of its
+ // siblings. Src2 may be invalid (absent or non-virtual) and is skipped.
+ for (Register Reg : {Src0, Src1, Src2, Vdst}) {
+ if (Reg.isValid() && Reg != VirtReg && !is_contained(Siblings, Reg))
+ Siblings.push_back(Reg);
+ }
+}
+
// VGPR bank-conflict avoidance for v_wmma_* operands on gfx11/gfx12.
//
// A WMMA reads src0/src1/src2 through per-bank read ports (bank = first VGPR
@@ -4179,20 +4213,24 @@ void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
const MachineFunction &MF,
const VirtRegMap *VRM) const {
const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const SIInstrInfo *TII = ST.getInstrInfo();
if (!VRM || !isVGPR(MRI, VirtReg))
return;
const SIMachineFunctionInfo &FuncInfo = *MF.getInfo<SIMachineFunctionInfo>();
- const DenseMap<Register, SmallVector<Register, 3>> &Siblings =
- FuncInfo.getWMMABankSiblings();
+ SmallVector<Register, 3> Siblings;
+ for (MachineInstr &MI : MRI.reg_nodbg_instructions(VirtReg)) {
+ if (!SIInstrInfo::isWMMA(MI))
+ continue;
+ recordWMMABankSiblings(VirtReg, MI, Siblings, TII, this, MRI);
+ }
- auto It = Siblings.find(VirtReg);
- if (It == Siblings.end())
+ if (Siblings.empty())
return;
// Banks already taken by allocated siblings.
unsigned UsedBankMask = 0;
- for (Register Sibling : It->second) {
+ for (Register Sibling : Siblings) {
Register Phys = Sibling;
if (Phys.isVirtual()) {
if (!VRM->hasPhys(Phys))
@@ -4226,7 +4264,7 @@ void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
// to very high registers, inflating the footprint and dropping occupancy on
// low-pressure functions (and triggering MSG_DEALLOC_VGPRS).
unsigned MaxVGPR = MaxIdx + W;
- if (unsigned Peak = FuncInfo.getWMMAPeakVGPRPressure()) {
+ if (unsigned Peak = FuncInfo.getPeakVGPRPressure()) {
unsigned Gran = AMDGPU::IsaInfo::getVGPRAllocGranule(
ST, FuncInfo.getDynamicVGPRBlockSize());
unsigned Want = alignTo(Peak, std::max(Gran, 1u));
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
index 4a8b2d52b3925..398ed287cdac5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
@@ -75,11 +75,11 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
@@ -150,11 +150,11 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index e3dfb6feab7c1..8e473bbd50649 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -14,58 +14,58 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v34, v0
-; COEXEC-NEXT: v_dual_mov_b32 v35, v0 :: v_dual_mov_b32 v36, v0
-; COEXEC-NEXT: v_dual_mov_b32 v37, v0 :: v_dual_mov_b32 v38, v0
-; COEXEC-NEXT: v_dual_mov_b32 v39, v0 :: v_dual_mov_b32 v40, v0
-; COEXEC-NEXT: v_dual_mov_b32 v41, v0 :: v_dual_mov_b32 v8, v0
+; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v50, v0
+; COEXEC-NEXT: v_dual_mov_b32 v51, v0 :: v_dual_mov_b32 v52, v0
+; COEXEC-NEXT: v_dual_mov_b32 v53, v0 :: v_dual_mov_b32 v54, v0
+; COEXEC-NEXT: v_dual_mov_b32 v55, v0 :: v_dual_mov_b32 v56, v0
+; COEXEC-NEXT: v_dual_mov_b32 v57, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v42, v0
-; COEXEC-NEXT: v_dual_mov_b32 v43, v0 :: v_dual_mov_b32 v44, v0
-; COEXEC-NEXT: v_dual_mov_b32 v45, v0 :: v_dual_mov_b32 v46, v0
+; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v58, v0
+; COEXEC-NEXT: v_dual_mov_b32 v59, v0 :: v_dual_mov_b32 v60, v0
+; COEXEC-NEXT: v_dual_mov_b32 v61, v0 :: v_dual_mov_b32 v62, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v47, v0
+; COEXEC-NEXT: v_mov_b32_e32 v63, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v48, v0
+; COEXEC-NEXT: v_mov_b32_e32 v64, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
-; COEXEC-NEXT: v_mov_b32_e32 v49, v0
+; COEXEC-NEXT: v_mov_b32_e32 v65, v0
; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB0_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; COEXEC-NEXT: v_mov_b32_e32 v32, s2
+; COEXEC-NEXT: v_mov_b32_e32 v48, s2
; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
-; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v32 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v32
-; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v32 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v32 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[54:57], v32 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[50:53], v32 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[58:61], v32 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[62:65], v32 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[70:73], v32 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[66:69], v32 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[78:81], v32 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[74:77], v32 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[86:89], v32 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v32 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v32 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v32 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v48
+; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v48 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v48 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[70:73], v48 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[66:69], v48 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v48 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v48 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v48 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v48 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[78:81], v48 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[74:77], v48 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:960
+; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v48 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v48 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v48 offset:768
; COEXEC-NEXT: s_wait_dscnt 0xc
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[24:31], v[16:23], v[42:49]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[24:31], v[16:23], v[58:65]
; COEXEC-NEXT: s_wait_dscnt 0x8
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[58:65], v[50:57], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[32:39], v[66:73], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[74:81], v[66:73], v[34:41]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[74:81], v[40:47], v[50:57]
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[82:89], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[24:31], v[16:23], v[42:49]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[58:65], v[50:57], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[74:81], v[66:73], v[34:41]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[24:31], v[16:23], v[58:65]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[32:39], v[66:73], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[74:81], v[40:47], v[50:57]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[82:89], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB0_1
; COEXEC-NEXT: ; %bb.2: ; %end
@@ -74,12 +74,12 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v16, v[46:49], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v16, v[42:45], s[0:1]
+; COEXEC-NEXT: global_store_b128 v16, v[62:65], s[0:1] offset:16
+; COEXEC-NEXT: global_store_b128 v16, v[58:61], s[0:1]
; COEXEC-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
; COEXEC-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v16, v[38:41], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v16, v[34:37], s[0:1] offset:256
+; COEXEC-NEXT: global_store_b128 v16, v[54:57], s[0:1] offset:272
+; COEXEC-NEXT: global_store_b128 v16, v[50:53], s[0:1] offset:256
; COEXEC-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
; COEXEC-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -95,57 +95,57 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v34, v0
-; GCN-NEXT: v_dual_mov_b32 v35, v0 :: v_dual_mov_b32 v36, v0
-; GCN-NEXT: v_dual_mov_b32 v37, v0 :: v_dual_mov_b32 v38, v0
-; GCN-NEXT: v_dual_mov_b32 v39, v0 :: v_dual_mov_b32 v40, v0
+; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v50, v0
+; GCN-NEXT: v_dual_mov_b32 v51, v0 :: v_dual_mov_b32 v52, v0
+; GCN-NEXT: v_dual_mov_b32 v53, v0 :: v_dual_mov_b32 v54, v0
+; GCN-NEXT: v_dual_mov_b32 v55, v0 :: v_dual_mov_b32 v56, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v41, v0 :: v_dual_mov_b32 v8, v0
+; GCN-NEXT: v_dual_mov_b32 v57, v0 :: v_dual_mov_b32 v8, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; GCN-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v42, v0
+; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v58, v0
; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
-; GCN-NEXT: v_dual_mov_b32 v43, v0 :: v_dual_mov_b32 v44, v0
-; GCN-NEXT: v_dual_mov_b32 v45, v0 :: v_dual_mov_b32 v46, v0
-; GCN-NEXT: v_dual_mov_b32 v47, v0 :: v_dual_mov_b32 v48, v0
-; GCN-NEXT: v_mov_b32_e32 v49, v0
+; GCN-NEXT: v_dual_mov_b32 v59, v0 :: v_dual_mov_b32 v60, v0
+; GCN-NEXT: v_dual_mov_b32 v61, v0 :: v_dual_mov_b32 v62, v0
+; GCN-NEXT: v_dual_mov_b32 v63, v0 :: v_dual_mov_b32 v64, v0
+; GCN-NEXT: v_mov_b32_e32 v65, v0
; GCN-NEXT: .LBB0_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_mov_b32_e32 v32, s2
+; GCN-NEXT: v_mov_b32_e32 v48, s2
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s2, s2, s1
-; GCN-NEXT: ds_load_tr16_b128 v[16:19], v32
-; GCN-NEXT: ds_load_tr16_b128 v[20:23], v32 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[24:27], v32 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[28:31], v32 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[50:53], v32 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[54:57], v32 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[58:61], v32 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[62:65], v32 offset:448
-; GCN-NEXT: ds_load_tr16_b128 v[66:69], v32 offset:512
-; GCN-NEXT: ds_load_tr16_b128 v[70:73], v32 offset:576
-; GCN-NEXT: ds_load_tr16_b128 v[74:77], v32 offset:640
-; GCN-NEXT: ds_load_tr16_b128 v[78:81], v32 offset:704
-; GCN-NEXT: ds_load_tr16_b128 v[82:85], v32 offset:768
-; GCN-NEXT: ds_load_tr16_b128 v[86:89], v32 offset:832
-; GCN-NEXT: ds_load_tr16_b128 v[90:93], v32 offset:896
-; GCN-NEXT: ds_load_tr16_b128 v[94:97], v32 offset:960
+; GCN-NEXT: ds_load_tr16_b128 v[16:19], v48
+; GCN-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[24:27], v48 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[28:31], v48 offset:192
+; GCN-NEXT: ds_load_tr16_b128 v[66:69], v48 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[70:73], v48 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[32:35], v48 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[36:39], v48 offset:448
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v48 offset:512
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v48 offset:576
+; GCN-NEXT: ds_load_tr16_b128 v[74:77], v48 offset:640
+; GCN-NEXT: ds_load_tr16_b128 v[78:81], v48 offset:704
+; GCN-NEXT: ds_load_tr16_b128 v[82:85], v48 offset:768
+; GCN-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:832
+; GCN-NEXT: ds_load_tr16_b128 v[90:93], v48 offset:896
+; GCN-NEXT: ds_load_tr16_b128 v[94:97], v48 offset:960
; GCN-NEXT: s_wait_dscnt 0xc
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[16:23], v[24:31], v[42:49]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
; GCN-NEXT: s_wait_dscnt 0x8
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[50:57], v[58:65], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x4
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[66:73], v[74:81], v[34:41]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[42:49], v[16:23], v[24:31], v[42:49]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[50:57], v[58:65], v[8:15]
-; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[34:41], v[66:73], v[74:81], v[34:41]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
+; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.2: ; %end
@@ -154,12 +154,12 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_mov_b32_e32 v16, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_store_b128 v16, v[46:49], s[0:1] offset:16
-; GCN-NEXT: global_store_b128 v16, v[42:45], s[0:1]
+; GCN-NEXT: global_store_b128 v16, v[62:65], s[0:1] offset:16
+; GCN-NEXT: global_store_b128 v16, v[58:61], s[0:1]
; GCN-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
; GCN-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
-; GCN-NEXT: global_store_b128 v16, v[38:41], s[0:1] offset:272
-; GCN-NEXT: global_store_b128 v16, v[34:37], s[0:1] offset:256
+; GCN-NEXT: global_store_b128 v16, v[54:57], s[0:1] offset:272
+; GCN-NEXT: global_store_b128 v16, v[50:53], s[0:1] offset:256
; GCN-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
; GCN-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -248,24 +248,24 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v66, v0
-; COEXEC-NEXT: v_dual_mov_b32 v67, v0 :: v_dual_mov_b32 v68, v0
-; COEXEC-NEXT: v_dual_mov_b32 v69, v0 :: v_dual_mov_b32 v70, v0
-; COEXEC-NEXT: v_dual_mov_b32 v71, v0 :: v_dual_mov_b32 v72, v0
-; COEXEC-NEXT: v_dual_mov_b32 v73, v0 :: v_dual_mov_b32 v8, v0
+; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v82, v0
+; COEXEC-NEXT: v_dual_mov_b32 v83, v0 :: v_dual_mov_b32 v84, v0
+; COEXEC-NEXT: v_dual_mov_b32 v85, v0 :: v_dual_mov_b32 v86, v0
+; COEXEC-NEXT: v_dual_mov_b32 v87, v0 :: v_dual_mov_b32 v88, v0
+; COEXEC-NEXT: v_dual_mov_b32 v89, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v74, v0
-; COEXEC-NEXT: v_dual_mov_b32 v75, v0 :: v_dual_mov_b32 v76, v0
-; COEXEC-NEXT: v_dual_mov_b32 v77, v0 :: v_dual_mov_b32 v78, v0
+; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v90, v0
+; COEXEC-NEXT: v_dual_mov_b32 v91, v0 :: v_dual_mov_b32 v92, v0
+; COEXEC-NEXT: v_dual_mov_b32 v93, v0 :: v_dual_mov_b32 v94, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v79, v0
+; COEXEC-NEXT: v_mov_b32_e32 v95, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v80, v0
+; COEXEC-NEXT: v_mov_b32_e32 v96, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
-; COEXEC-NEXT: v_mov_b32_e32 v81, v0
+; COEXEC-NEXT: v_mov_b32_e32 v97, v0
; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB1_1: ; %loop
@@ -273,64 +273,64 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: s_add_co_i32 s7, s2, s6
; COEXEC-NEXT: s_add_co_i32 s8, s3, s6
; COEXEC-NEXT: s_add_co_i32 s6, s6, s1
-; COEXEC-NEXT: v_mov_b32_e32 v64, s7
+; COEXEC-NEXT: v_mov_b32_e32 v80, s7
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; COEXEC-NEXT: v_mov_b32_e32 v65, s8
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v64
-; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v64 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v65
-; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v65 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[86:89], v64 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v64 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v65 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v65 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v64 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v64 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v65 offset:576
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v65 offset:512
-; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v64 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v64 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v65 offset:832
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v65 offset:768
-; COEXEC-NEXT: ds_load_tr16_b128 v[98:101], v64 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[102:105], v64 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[106:109], v64 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[110:113], v64 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[114:117], v64 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[118:121], v64 offset:704
-; COEXEC-NEXT: ds_load_tr16_b128 v[122:125], v64 offset:896
+; COEXEC-NEXT: v_mov_b32_e32 v81, s8
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v80
+; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v80 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v81
+; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v81 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[102:105], v80 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[98:101], v80 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v81 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v81 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v80 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v80 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v81 offset:576
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v81 offset:512
+; COEXEC-NEXT: ds_load_tr16_b128 v[110:113], v80 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[106:109], v80 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v81 offset:832
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v81 offset:768
+; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v80 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v80 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[114:117], v80 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[118:121], v80 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v80 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v80 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[122:125], v80 offset:896
; COEXEC-NEXT: s_wait_dscnt 0x13
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[16:23], v[24:31], v[74:81]
-; COEXEC-NEXT: ds_load_tr16_b128 v[126:129], v64 offset:960
-; COEXEC-NEXT: ds_load_tr16_b128 v[130:133], v65 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[134:137], v65 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[138:141], v65 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[142:145], v65 offset:448
-; COEXEC-NEXT: ds_load_tr16_b128 v[146:149], v65 offset:640
-; COEXEC-NEXT: ds_load_tr16_b128 v[150:153], v65 offset:704
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[90:97], v[16:23], v[24:31], v[90:97]
+; COEXEC-NEXT: ds_load_tr16_b128 v[126:129], v80 offset:960
+; COEXEC-NEXT: ds_load_tr16_b128 v[130:133], v81 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[134:137], v81 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[138:141], v81 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[142:145], v81 offset:448
+; COEXEC-NEXT: ds_load_tr16_b128 v[146:149], v81 offset:640
+; COEXEC-NEXT: ds_load_tr16_b128 v[150:153], v81 offset:704
; COEXEC-NEXT: s_wait_dscnt 0x16
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[82:89], v[32:39], v[8:15]
-; COEXEC-NEXT: ds_load_tr16_b128 v[154:157], v65 offset:896
-; COEXEC-NEXT: ds_load_tr16_b128 v[158:161], v65 offset:960
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[98:105], v[32:39], v[8:15]
+; COEXEC-NEXT: ds_load_tr16_b128 v[154:157], v81 offset:896
+; COEXEC-NEXT: ds_load_tr16_b128 v[158:161], v81 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x14
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[40:47], v[48:55], v[66:73]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[82:89], v[40:47], v[48:55], v[82:89]
; COEXEC-NEXT: s_wait_dscnt 0x10
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[56:63], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[16:23], v[24:31], v[74:81]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[82:89], v[32:39], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[40:47], v[48:55], v[66:73]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[56:63], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[106:113], v[56:63], v[0:7]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[90:97], v[16:23], v[24:31], v[90:97]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[98:105], v[32:39], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[82:89], v[40:47], v[48:55], v[82:89]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[106:113], v[56:63], v[0:7]
; COEXEC-NEXT: s_wait_dscnt 0x6
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[98:105], v[130:137], v[74:81]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[90:97], v[64:71], v[130:137], v[90:97]
; COEXEC-NEXT: s_wait_dscnt 0x4
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[106:113], v[138:145], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[114:121], v[138:145], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x2
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[114:121], v[146:153], v[66:73]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[82:89], v[72:79], v[146:153], v[82:89]
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[122:129], v[154:161], v[0:7]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[74:81], v[98:105], v[130:137], v[74:81]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[106:113], v[138:145], v[8:15]
-; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[66:73], v[114:121], v[146:153], v[66:73]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[90:97], v[64:71], v[130:137], v[90:97]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[114:121], v[138:145], v[8:15]
+; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[82:89], v[72:79], v[146:153], v[82:89]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[122:129], v[154:161], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
@@ -338,12 +338,12 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v16, v[78:81], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v16, v[74:77], s[0:1]
+; COEXEC-NEXT: global_store_b128 v16, v[94:97], s[0:1] offset:16
+; COEXEC-NEXT: global_store_b128 v16, v[90:93], s[0:1]
; COEXEC-NEXT: global_store_b128 v16, v[12:15], s[0:1] offset:144
; COEXEC-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v16, v[70:73], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v16, v[66:69], s[0:1] offset:256
+; COEXEC-NEXT: global_store_b128 v16, v[86:89], s[0:1] offset:272
+; COEXEC-NEXT: global_store_b128 v16, v[82:85], s[0:1] offset:256
; COEXEC-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:400
; COEXEC-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
index 00a6bb4dd136c..4bb88e284b1e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
@@ -75,11 +75,11 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
@@ -150,11 +150,11 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
diff --git a/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll b/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
index 0503fa6ae20a5..5ced1ec7f7ce2 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
@@ -151,11 +151,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_unsigned(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_unsigned:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -171,11 +171,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_signed(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_signed:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[0,1,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -191,11 +191,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_unsigned(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_unsigned:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,0,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -211,11 +211,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_signed(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_signed:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,1,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -231,11 +231,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_unsigned_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_unsigned_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -251,11 +251,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_signed_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_signed_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -271,11 +271,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_unsigned_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_unsigned_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -291,11 +291,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_signed_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_signed_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
+; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
diff --git a/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll b/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
index 138d80d16c52e..cccdc88ab7aa4 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
@@ -23,9 +23,9 @@ declare <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v4i32.v2i32(i1 immarg, <2 x
define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<16 x half> %A, <16 x half> %B, <4 x float> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f32_16x16x16_f16:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f32_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f32_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f32_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -41,9 +41,9 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x16_bf16(<16 x i16> %A, <16 x i16> %B, <4 x float> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f32_16x16x16_bf16:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -59,9 +59,9 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_lo(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f16_16x16x16_f16_lo:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f16_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f16_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -75,9 +75,9 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_hi(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f16_16x16x16_f16_hi:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f16_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
+; W64-NEXT: v_wmma_f16_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19] op_sel:[0,0,1]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -93,9 +93,9 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_lo(<16 x i16> %A, <16 x i16> %B, <8 x i16> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_bf16_16x16x16_bf16_lo:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -109,9 +109,9 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_hi(<16 x i16> %A, <16 x i16> %B, <8 x i16> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_bf16_16x16x16_bf16_hi:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
+; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19] op_sel:[0,0,1]
-; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
+; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
>From 8dcd57ca695f1cfa70ac7f8394f4c2780418b465 Mon Sep 17 00:00:00 2001
From: shore <372660931 at qq.com>
Date: Thu, 25 Jun 2026 19:59:14 +0800
Subject: [PATCH 3/6] update conflict logic based on test results
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 53 ++-
.../AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll | 12 +-
.../CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll | 12 +-
...schedule-regpressure-ilp-metric-spills.mir | 7 +-
llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll | 425 +++++++++---------
llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll | 48 +-
llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll | 24 +-
7 files changed, 300 insertions(+), 281 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 091ce000632a8..8958a43eaad0a 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4169,26 +4169,29 @@ static void recordWMMABankSiblings(Register VirtReg, const MachineInstr &MI,
// hint to WMMAs whose matrix source operands are a multiple of 128 bits.
Register Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)->getReg();
Register Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)->getReg();
- if (TRI->getRegSizeInBits(Src0, MRI) % 128 != 0 ||
- TRI->getRegSizeInBits(Src1, MRI) % 128 != 0)
+ Register Vdst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
+ if (!TRI->isVGPR(MRI, Src0) || !TRI->isVGPR(MRI, Src1))
return;
- Register Src2, Vdst;
+ auto saveSiblings = [&](SmallVector<Register, 3> Regs) {
+ if (!is_contained(Regs, VirtReg))
+ return;
+ for (Register Reg : Regs) {
+ if (Reg != VirtReg && !is_contained(Siblings, Reg))
+ Siblings.push_back(Reg);
+ }
+ };
+
+ // Src2 == Dst :=> 2 cycle penalty for bank convlict between src0, src1, src2
+ // Src2 != Dst :=> 1 cycle penalty for bank convlict between src0, src1
const MachineOperand *MOSrc2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
- if (MOSrc2 && MOSrc2->isReg() && MOSrc2->getReg().isVirtual())
- Src2 = MOSrc2->getReg();
- const MachineOperand *MODst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
- if (MODst && MODst->isReg() && MODst->getReg().isVirtual())
- Vdst = MODst->getReg();
-
- // Record, for each matrix operand, the other matrix operands of this WMMA so
- // SIRegisterInfo::getRegAllocationHints can steer them onto distinct VGPR
- // banks. A register used by several WMMAs accumulates the union of its
- // siblings. Src2 may be invalid (absent or non-virtual) and is skipped.
- for (Register Reg : {Src0, Src1, Src2, Vdst}) {
- if (Reg.isValid() && Reg != VirtReg && !is_contained(Siblings, Reg))
- Siblings.push_back(Reg);
+ if (MOSrc2 && MOSrc2->isReg() && MOSrc2->getReg().isVirtual() &&
+ MOSrc2->getReg() == Vdst) {
+ saveSiblings({Src0, Src1, Vdst});
+ return;
}
+
+ saveSiblings({Src0, Src1});
}
// VGPR bank-conflict avoidance for v_wmma_* operands on gfx11/gfx12.
@@ -4257,6 +4260,18 @@ void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
MaxIdx = std::max(MaxIdx, Idx);
}
+ // Precompute once per function:
+ BitVector CalleeSavedUnits(getNumRegUnits());
+ for (const MCPhysReg *CSR = MRI.getCalleeSavedRegs(); CSR && *CSR; ++CSR)
+ for (MCRegUnit U : regunits(*CSR))
+ CalleeSavedUnits.set(static_cast<unsigned>(U));
+ // Then for a candidate physreg P (the tuple at Start):
+ auto isCalleeSaved = [&](MCRegister P) {
+ return any_of(regunits(P), [&](MCRegUnit U) {
+ return CalleeSavedUnits.test(static_cast<unsigned>(U));
+ });
+ };
+
// Anchor the candidate window to the function's actual VGPR footprint: peak
// WMMA-region pressure, rounded up to the allocation granule and capped by
// the VGPR budget. Without this the window is the top of the whole VGPR file
@@ -4279,11 +4294,11 @@ void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
unsigned BankSize = N * W;
for (unsigned n = 0; n < N; n++) {
for (unsigned Bank = 0; Bank < 4; Bank++) {
- if (((UsedBankMask >> Bank) & 1) != 0)
+ unsigned Start = n * W + Bank * (BankSize + 1);
+ if (((UsedBankMask >> (Start % 4)) & 1) != 0)
continue;
- unsigned Start = Bank * BankSize + Bank + n * W;
auto RIt = IdxToReg.find(Start);
- if (RIt != IdxToReg.end())
+ if (RIt != IdxToReg.end() && !isCalleeSaved(RIt->second))
Hints.push_back(RIt->second);
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
index 398ed287cdac5..4a8b2d52b3925 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
@@ -75,11 +75,11 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
-; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
@@ -150,11 +150,11 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
-; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
+; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
index 4bb88e284b1e4..00a6bb4dd136c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma_32.ll
@@ -75,11 +75,11 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x half> %B.0, <16 x half> %A.1, <16 x half> %B.1, <16 x half> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_f16_16x16x16_f16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_f16_16x16x16_f16 v[51:58], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
@@ -150,11 +150,11 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16 x i16> %B.0, <16 x i16> %A.1, <16 x i16> %B.1, <16 x i16> %C, ptr addrspace(1) %out.0, ptr addrspace(1) %out.1) {
; W32-LABEL: test_wmma_bf16_16x16x16_bf16_untied:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[51:58], v[0:7], v[8:15], v[32:39]
+; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[40:41], v[55:58], off offset:16
-; W32-NEXT: global_store_b128 v[40:41], v[51:54], off
+; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
+; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index aa0d1fe45e9a8..00e32abcf7801 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -14,12 +14,17 @@
# GCN: S_ENDPGM
# GCN-GCNTRACKER-LABEL: name: no_sched_metric_due_to_spills
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_V256_SAVE
# GCN-GCNTRACKER: S_ENDPGM
# When using the GCN Trackers, the scheduler is able to acieve desired occupancy without running high-RP-reschedule stage. However, the RP is still high,
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
index de2922e811caf..bde3755c1862f 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
@@ -198,54 +198,54 @@ define amdgpu_kernel void @wmma_f32_3x_high_pressure(ptr addrspace(1) %pa, ptr a
; GCN-NEXT: s_load_b256 s[24:31], s[46:47], 0x20
; GCN-NEXT: s_load_b256 s[36:43], s[44:45], 0x40
; GCN-NEXT: s_load_b256 s[44:51], s[46:47], 0x40
+; GCN-NEXT: s_mov_b32 s59, s52
; GCN-NEXT: s_mov_b32 s53, s52
; GCN-NEXT: s_mov_b32 s54, s52
; GCN-NEXT: s_mov_b32 s55, s52
; GCN-NEXT: s_mov_b32 s56, s52
; GCN-NEXT: s_mov_b32 s57, s52
; GCN-NEXT: s_mov_b32 s58, s52
-; GCN-NEXT: s_mov_b32 s59, s52
-; GCN-NEXT: v_dual_mov_b32 v0, s52 :: v_dual_mov_b32 v3, s55
-; GCN-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s53
-; GCN-NEXT: v_dual_mov_b32 v2, s54 :: v_dual_mov_b32 v5, s57
-; GCN-NEXT: v_dual_mov_b32 v4, s56 :: v_dual_mov_b32 v7, s59
+; GCN-NEXT: v_dual_mov_b32 v33, 0 :: v_dual_mov_b32 v0, s52
+; GCN-NEXT: v_mov_b32_e32 v7, s59
+; GCN-NEXT: v_dual_mov_b32 v1, s53 :: v_dual_mov_b32 v2, s54
+; GCN-NEXT: v_dual_mov_b32 v3, s55 :: v_dual_mov_b32 v4, s56
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v21, s4
-; GCN-NEXT: v_dual_mov_b32 v18, s1 :: v_dual_mov_b32 v41, s15
+; GCN-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v41, s15
+; GCN-NEXT: v_dual_mov_b32 v38, s12 :: v_dual_mov_b32 v5, s57
; GCN-NEXT: v_dual_mov_b32 v6, s58 :: v_dual_mov_b32 v23, s6
-; GCN-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v19, s2
-; GCN-NEXT: v_dual_mov_b32 v20, s3 :: v_dual_mov_b32 v17, s0
+; GCN-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v21, s4
+; GCN-NEXT: v_dual_mov_b32 v20, s3 :: v_dual_mov_b32 v19, s2
+; GCN-NEXT: v_dual_mov_b32 v18, s1 :: v_dual_mov_b32 v17, s0
; GCN-NEXT: v_dual_mov_b32 v40, s14 :: v_dual_mov_b32 v39, s13
-; GCN-NEXT: v_dual_mov_b32 v38, s12 :: v_dual_mov_b32 v37, s11
-; GCN-NEXT: v_dual_mov_b32 v36, s10 :: v_dual_mov_b32 v35, s9
-; GCN-NEXT: v_mov_b32_e32 v34, s8
-; GCN-NEXT: v_dual_mov_b32 v58, s23 :: v_dual_mov_b32 v55, s20
-; GCN-NEXT: v_dual_mov_b32 v32, s31 :: v_dual_mov_b32 v29, s28
-; GCN-NEXT: v_dual_mov_b32 v57, s22 :: v_dual_mov_b32 v56, s21
-; GCN-NEXT: v_dual_mov_b32 v53, s18 :: v_dual_mov_b32 v54, s19
-; GCN-NEXT: v_dual_mov_b32 v51, s16 :: v_dual_mov_b32 v52, s17
-; GCN-NEXT: v_dual_mov_b32 v31, s30 :: v_dual_mov_b32 v30, s29
-; GCN-NEXT: v_dual_mov_b32 v27, s26 :: v_dual_mov_b32 v28, s27
-; GCN-NEXT: v_dual_mov_b32 v25, s24 :: v_dual_mov_b32 v26, s25
-; GCN-NEXT: v_dual_mov_b32 v49, s43 :: v_dual_mov_b32 v66, s51
-; GCN-NEXT: v_dual_mov_b32 v63, s48 :: v_dual_mov_b32 v48, s42
-; GCN-NEXT: v_dual_mov_b32 v47, s41 :: v_dual_mov_b32 v46, s40
-; GCN-NEXT: v_dual_mov_b32 v45, s39 :: v_dual_mov_b32 v44, s38
-; GCN-NEXT: v_dual_mov_b32 v43, s37 :: v_dual_mov_b32 v42, s36
-; GCN-NEXT: v_dual_mov_b32 v65, s50 :: v_dual_mov_b32 v64, s49
-; GCN-NEXT: v_dual_mov_b32 v61, s46 :: v_dual_mov_b32 v62, s47
-; GCN-NEXT: v_dual_mov_b32 v59, s44 :: v_dual_mov_b32 v60, s45
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[17:24], v[34:41], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[34:41], v[51:58], v[25:32], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v36, s10 :: v_dual_mov_b32 v37, s11
+; GCN-NEXT: v_dual_mov_b32 v34, s8 :: v_dual_mov_b32 v35, s9
+; GCN-NEXT: v_dual_mov_b32 v58, s23 :: v_dual_mov_b32 v51, s16
+; GCN-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v57, s22
+; GCN-NEXT: v_dual_mov_b32 v56, s21 :: v_dual_mov_b32 v55, s20
+; GCN-NEXT: v_dual_mov_b32 v54, s19 :: v_dual_mov_b32 v53, s18
+; GCN-NEXT: v_dual_mov_b32 v52, s17 :: v_dual_mov_b32 v9, s25
+; GCN-NEXT: v_dual_mov_b32 v10, s26 :: v_dual_mov_b32 v11, s27
+; GCN-NEXT: v_dual_mov_b32 v12, s28 :: v_dual_mov_b32 v13, s29
+; GCN-NEXT: v_dual_mov_b32 v14, s30 :: v_dual_mov_b32 v15, s31
+; GCN-NEXT: v_dual_mov_b32 v25, s36 :: v_dual_mov_b32 v28, s39
+; GCN-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v42, s44
+; GCN-NEXT: v_dual_mov_b32 v26, s37 :: v_dual_mov_b32 v27, s38
+; GCN-NEXT: v_dual_mov_b32 v30, s41 :: v_dual_mov_b32 v29, s40
+; GCN-NEXT: v_dual_mov_b32 v32, s43 :: v_dual_mov_b32 v43, s45
+; GCN-NEXT: v_dual_mov_b32 v44, s46 :: v_dual_mov_b32 v45, s47
+; GCN-NEXT: v_dual_mov_b32 v46, s48 :: v_dual_mov_b32 v47, s49
+; GCN-NEXT: v_dual_mov_b32 v48, s50 :: v_dual_mov_b32 v49, s51
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[59:66], v[17:24], v[34:41], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[16:23], v[51:58], v[8:15], v[0:7]
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[42:49], v[59:66], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[25:32], v[42:49], v[0:7]
; GCN-NEXT: s_clause 0x5
-; GCN-NEXT: global_store_b128 v16, v[12:15], s[34:35] offset:16
-; GCN-NEXT: global_store_b128 v16, v[8:11], s[34:35]
-; GCN-NEXT: global_store_b128 v16, v[38:41], s[34:35] offset:48
-; GCN-NEXT: global_store_b128 v16, v[34:37], s[34:35] offset:32
-; GCN-NEXT: global_store_b128 v16, v[4:7], s[34:35] offset:80
-; GCN-NEXT: global_store_b128 v16, v[0:3], s[34:35] offset:64
+; GCN-NEXT: global_store_b128 v33, v[63:66], s[34:35] offset:16
+; GCN-NEXT: global_store_b128 v33, v[59:62], s[34:35]
+; GCN-NEXT: global_store_b128 v33, v[20:23], s[34:35] offset:48
+; GCN-NEXT: global_store_b128 v33, v[16:19], s[34:35] offset:32
+; GCN-NEXT: global_store_b128 v33, v[4:7], s[34:35] offset:80
+; GCN-NEXT: global_store_b128 v33, v[0:3], s[34:35] offset:64
; GCN-NEXT: s_endpgm
%pa0 = getelementptr <16 x half>, ptr addrspace(1) %pa, i64 0
%a0 = load <16 x half>, ptr addrspace(1) %pa0
@@ -278,7 +278,7 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-LABEL: wmma_f32_12x_pressure:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GCN-NEXT: ; implicit-def: $vgpr65 : SGPR spill to VGPR lane
+; GCN-NEXT: ; implicit-def: $vgpr139 : SGPR spill to VGPR lane
; GCN-NEXT: s_mov_b32 s8, 0
; GCN-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GCN-NEXT: s_mov_b32 s14, s8
@@ -289,7 +289,7 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: s_mov_b32 s13, s8
; GCN-NEXT: s_mov_b32 s15, s8
; GCN-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
-; GCN-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v1, s9
+; GCN-NEXT: v_dual_mov_b32 v98, 0 :: v_dual_mov_b32 v1, s9
; GCN-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
; GCN-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -307,215 +307,214 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x40
; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x40
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v49, s68 :: v_dual_mov_b32 v56, s75
-; GCN-NEXT: v_dual_mov_b32 v89, s83 :: v_dual_mov_b32 v122, s91
-; GCN-NEXT: v_mov_b32_e32 v119, s88
-; GCN-NEXT: v_writelane_b32 v65, s16, 0
+; GCN-NEXT: v_dual_mov_b32 v114, s75 :: v_dual_mov_b32 v111, s72
+; GCN-NEXT: v_dual_mov_b32 v16, s76 :: v_dual_mov_b32 v19, s79
+; GCN-NEXT: v_writelane_b32 v139, s16, 0
; GCN-NEXT: v_dual_mov_b32 v40, s31 :: v_dual_mov_b32 v37, s28
; GCN-NEXT: v_dual_mov_b32 v39, s30 :: v_dual_mov_b32 v38, s29
; GCN-NEXT: v_mov_b32_e32 v35, s26
-; GCN-NEXT: v_writelane_b32 v65, s17, 1
+; GCN-NEXT: v_writelane_b32 v139, s17, 1
; GCN-NEXT: v_dual_mov_b32 v36, s27 :: v_dual_mov_b32 v33, s24
; GCN-NEXT: v_dual_mov_b32 v34, s25 :: v_dual_mov_b32 v73, s51
-; GCN-NEXT: v_writelane_b32 v65, s18, 2
+; GCN-NEXT: v_writelane_b32 v139, s18, 2
; GCN-NEXT: s_load_b256 s[24:31], s[0:1], 0xc0
; GCN-NEXT: v_dual_mov_b32 v106, s43 :: v_dual_mov_b32 v103, s40
; GCN-NEXT: v_dual_mov_b32 v66, s44 :: v_dual_mov_b32 v105, s42
-; GCN-NEXT: v_writelane_b32 v65, s19, 3
+; GCN-NEXT: v_writelane_b32 v139, s19, 3
; GCN-NEXT: v_dual_mov_b32 v104, s41 :: v_dual_mov_b32 v101, s38
; GCN-NEXT: v_dual_mov_b32 v102, s39 :: v_dual_mov_b32 v99, s36
-; GCN-NEXT: v_writelane_b32 v65, s20, 4
-; GCN-NEXT: v_dual_mov_b32 v114, s67 :: v_dual_mov_b32 v111, s64
+; GCN-NEXT: v_writelane_b32 v139, s20, 4
; GCN-NEXT: v_dual_mov_b32 v72, s50 :: v_dual_mov_b32 v71, s49
-; GCN-NEXT: v_writelane_b32 v65, s21, 5
; GCN-NEXT: v_dual_mov_b32 v70, s48 :: v_dual_mov_b32 v69, s47
+; GCN-NEXT: v_writelane_b32 v139, s21, 5
; GCN-NEXT: v_dual_mov_b32 v68, s46 :: v_dual_mov_b32 v67, s45
-; GCN-NEXT: v_writelane_b32 v65, s22, 6
-; GCN-NEXT: v_dual_mov_b32 v112, s65 :: v_dual_mov_b32 v109, s62
-; GCN-NEXT: v_dual_mov_b32 v110, s63 :: v_dual_mov_b32 v107, s60
-; GCN-NEXT: v_writelane_b32 v65, s23, 7
+; GCN-NEXT: v_dual_mov_b32 v82, s92 :: v_dual_mov_b32 v85, s95
+; GCN-NEXT: v_writelane_b32 v139, s22, 6
+; GCN-NEXT: v_dual_mov_b32 v22, s82 :: v_dual_mov_b32 v49, s84
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[24:31], v[33:40], v[66:73], v[0:7]
+; GCN-NEXT: v_writelane_b32 v139, s23, 7
; GCN-NEXT: s_load_b256 s[8:15], s[0:1], 0xa0
; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0xa0
-; GCN-NEXT: v_mov_b32_e32 v108, s61
-; GCN-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v57, s92
+; GCN-NEXT: v_mov_b32_e32 v88, s98
+; GCN-NEXT: v_dual_mov_b32 v112, s73 :: v_dual_mov_b32 v109, s70
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_writelane_b32 v65, s24, 8
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[123:130], v[33:40], v[66:73], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v50, s69 :: v_dual_mov_b32 v51, s70
-; GCN-NEXT: v_writelane_b32 v65, s25, 9
-; GCN-NEXT: v_dual_mov_b32 v52, s71 :: v_dual_mov_b32 v53, s72
-; GCN-NEXT: v_dual_mov_b32 v54, s73 :: v_dual_mov_b32 v55, s74
-; GCN-NEXT: v_writelane_b32 v65, s26, 10
-; GCN-NEXT: v_dual_mov_b32 v88, s82 :: v_dual_mov_b32 v87, s81
-; GCN-NEXT: v_dual_mov_b32 v86, s80 :: v_dual_mov_b32 v85, s79
-; GCN-NEXT: v_writelane_b32 v65, s27, 11
-; GCN-NEXT: v_dual_mov_b32 v84, s78 :: v_dual_mov_b32 v83, s77
-; GCN-NEXT: v_dual_mov_b32 v82, s76 :: v_dual_mov_b32 v121, s90
-; GCN-NEXT: v_writelane_b32 v65, s28, 12
-; GCN-NEXT: v_mov_b32_e32 v64, s99
-; GCN-NEXT: v_dual_mov_b32 v120, s89 :: v_dual_mov_b32 v117, s86
-; GCN-NEXT: v_dual_mov_b32 v118, s87 :: v_dual_mov_b32 v115, s84
-; GCN-NEXT: v_writelane_b32 v65, s29, 13
-; GCN-NEXT: v_dual_mov_b32 v58, s93 :: v_dual_mov_b32 v59, s94
-; GCN-NEXT: v_dual_mov_b32 v60, s95 :: v_dual_mov_b32 v61, s96
-; GCN-NEXT: v_writelane_b32 v65, s30, 14
-; GCN-NEXT: v_dual_mov_b32 v62, s97 :: v_dual_mov_b32 v63, s98
-; GCN-NEXT: v_dual_mov_b32 v81, s59 :: v_dual_mov_b32 v80, s58
-; GCN-NEXT: v_mov_b32_e32 v79, s57
-; GCN-NEXT: v_writelane_b32 v65, s31, 15
+; GCN-NEXT: v_writelane_b32 v139, s24, 8
+; GCN-NEXT: v_dual_mov_b32 v110, s71 :: v_dual_mov_b32 v107, s68
+; GCN-NEXT: v_dual_mov_b32 v108, s69 :: v_dual_mov_b32 v17, s77
+; GCN-NEXT: v_writelane_b32 v139, s25, 9
+; GCN-NEXT: v_dual_mov_b32 v18, s78 :: v_dual_mov_b32 v21, s81
+; GCN-NEXT: v_dual_mov_b32 v20, s80 :: v_dual_mov_b32 v23, s83
+; GCN-NEXT: v_writelane_b32 v139, s26, 10
+; GCN-NEXT: v_dual_mov_b32 v50, s85 :: v_dual_mov_b32 v51, s86
+; GCN-NEXT: v_dual_mov_b32 v52, s87 :: v_dual_mov_b32 v53, s88
+; GCN-NEXT: v_writelane_b32 v139, s27, 11
+; GCN-NEXT: v_dual_mov_b32 v73, s15 :: v_dual_mov_b32 v54, s89
+; GCN-NEXT: v_dual_mov_b32 v55, s90 :: v_dual_mov_b32 v56, s91
+; GCN-NEXT: v_mov_b32_e32 v83, s93
+; GCN-NEXT: v_writelane_b32 v139, s28, 12
+; GCN-NEXT: v_dual_mov_b32 v84, s94 :: v_dual_mov_b32 v87, s97
+; GCN-NEXT: v_dual_mov_b32 v86, s96 :: v_dual_mov_b32 v89, s99
+; GCN-NEXT: v_writelane_b32 v139, s29, 13
+; GCN-NEXT: v_dual_mov_b32 v72, s14 :: v_dual_mov_b32 v71, s13
+; GCN-NEXT: v_dual_mov_b32 v70, s12 :: v_dual_mov_b32 v69, s11
+; GCN-NEXT: v_writelane_b32 v139, s30, 14
+; GCN-NEXT: v_dual_mov_b32 v68, s10 :: v_dual_mov_b32 v67, s9
+; GCN-NEXT: v_dual_mov_b32 v66, s8 :: v_dual_mov_b32 v41, s52
+; GCN-NEXT: v_writelane_b32 v139, s31, 15
; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xc0
-; GCN-NEXT: v_dual_mov_b32 v78, s56 :: v_dual_mov_b32 v77, s55
-; GCN-NEXT: v_dual_mov_b32 v76, s54 :: v_dual_mov_b32 v75, s53
-; GCN-NEXT: v_dual_mov_b32 v74, s52 :: v_dual_mov_b32 v113, s66
+; GCN-NEXT: v_dual_mov_b32 v48, s59 :: v_dual_mov_b32 v81, s67
+; GCN-NEXT: v_dual_mov_b32 v42, s53 :: v_dual_mov_b32 v43, s54
+; GCN-NEXT: v_dual_mov_b32 v44, s55 :: v_dual_mov_b32 v45, s56
+; GCN-NEXT: v_dual_mov_b32 v46, s57 :: v_dual_mov_b32 v47, s58
; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x100
+; GCN-NEXT: v_dual_mov_b32 v80, s66 :: v_dual_mov_b32 v79, s65
+; GCN-NEXT: v_dual_mov_b32 v78, s64 :: v_dual_mov_b32 v77, s63
+; GCN-NEXT: v_dual_mov_b32 v76, s62 :: v_dual_mov_b32 v75, s61
+; GCN-NEXT: v_dual_mov_b32 v74, s60 :: v_dual_mov_b32 v113, s74
; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x100
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[33:40], v[74:81], v[107:114], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v114, s23 :: v_dual_mov_b32 v111, s20
-; GCN-NEXT: v_dual_mov_b32 v113, s22 :: v_dual_mov_b32 v112, s21
-; GCN-NEXT: v_dual_mov_b32 v109, s18 :: v_dual_mov_b32 v110, s19
-; GCN-NEXT: v_mov_b32_e32 v107, s16
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_writelane_b32 v65, s24, 16
-; GCN-NEXT: v_writelane_b32 v65, s25, 17
-; GCN-NEXT: v_dual_mov_b32 v97, s67 :: v_dual_mov_b32 v96, s66
-; GCN-NEXT: v_mov_b32_e32 v95, s65
-; GCN-NEXT: v_writelane_b32 v65, s26, 18
-; GCN-NEXT: v_dual_mov_b32 v94, s64 :: v_dual_mov_b32 v93, s63
-; GCN-NEXT: v_dual_mov_b32 v92, s62 :: v_dual_mov_b32 v91, s61
-; GCN-NEXT: v_writelane_b32 v65, s27, 19
-; GCN-NEXT: v_writelane_b32 v65, s28, 20
-; GCN-NEXT: v_writelane_b32 v65, s29, 21
-; GCN-NEXT: v_writelane_b32 v65, s30, 22
-; GCN-NEXT: v_writelane_b32 v65, s31, 23
+; GCN-NEXT: v_writelane_b32 v139, s24, 16
+; GCN-NEXT: v_writelane_b32 v139, s25, 17
+; GCN-NEXT: v_writelane_b32 v139, s26, 18
+; GCN-NEXT: v_writelane_b32 v139, s27, 19
+; GCN-NEXT: v_dual_mov_b32 v122, s67 :: v_dual_mov_b32 v119, s64
+; GCN-NEXT: v_writelane_b32 v139, s28, 20
+; GCN-NEXT: v_dual_mov_b32 v121, s66 :: v_dual_mov_b32 v120, s65
+; GCN-NEXT: v_dual_mov_b32 v117, s62 :: v_dual_mov_b32 v118, s63
+; GCN-NEXT: v_mov_b32_e32 v115, s60
+; GCN-NEXT: v_writelane_b32 v139, s29, 21
+; GCN-NEXT: v_writelane_b32 v139, s30, 22
+; GCN-NEXT: v_writelane_b32 v139, s31, 23
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT: v_readlane_b32 s24, v65, 0
-; GCN-NEXT: v_readlane_b32 s31, v65, 7
-; GCN-NEXT: v_readlane_b32 s25, v65, 1
-; GCN-NEXT: v_readlane_b32 s26, v65, 2
-; GCN-NEXT: v_readlane_b32 s27, v65, 3
-; GCN-NEXT: v_readlane_b32 s28, v65, 4
-; GCN-NEXT: v_readlane_b32 s29, v65, 5
-; GCN-NEXT: v_readlane_b32 s30, v65, 6
-; GCN-NEXT: v_mov_b32_e32 v48, s31
-; GCN-NEXT: v_mov_b32_e32 v100, s37
+; GCN-NEXT: v_readlane_b32 s24, v139, 0
+; GCN-NEXT: v_readlane_b32 s30, v139, 6
+; GCN-NEXT: v_readlane_b32 s25, v139, 1
+; GCN-NEXT: v_readlane_b32 s26, v139, 2
+; GCN-NEXT: v_readlane_b32 s27, v139, 3
+; GCN-NEXT: v_readlane_b32 s28, v139, 4
+; GCN-NEXT: v_readlane_b32 s29, v139, 5
+; GCN-NEXT: v_readlane_b32 s31, v139, 7
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GCN-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v11, s27
+; GCN-NEXT: v_mov_b32_e32 v14, s30
+; GCN-NEXT: v_dual_mov_b32 v100, s37 :: v_dual_mov_b32 v9, s25
; GCN-NEXT: s_load_b256 s[36:43], s[0:1], 0xe0
-; GCN-NEXT: v_dual_mov_b32 v46, s29 :: v_dual_mov_b32 v43, s26
-; GCN-NEXT: v_mov_b32_e32 v47, s30
-; GCN-NEXT: v_dual_mov_b32 v45, s28 :: v_dual_mov_b32 v44, s27
-; GCN-NEXT: v_dual_mov_b32 v41, s24 :: v_dual_mov_b32 v42, s25
+; GCN-NEXT: v_dual_mov_b32 v10, s26 :: v_dual_mov_b32 v13, s29
+; GCN-NEXT: v_dual_mov_b32 v12, s28 :: v_dual_mov_b32 v15, s31
; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xe0
-; GCN-NEXT: v_readlane_b32 s44, v65, 8
-; GCN-NEXT: v_readlane_b32 s45, v65, 9
-; GCN-NEXT: v_readlane_b32 s46, v65, 10
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[66:73], v[99:106], v[41:48], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v48, s15 :: v_dual_mov_b32 v45, s12
-; GCN-NEXT: v_dual_mov_b32 v47, s14 :: v_dual_mov_b32 v46, s13
-; GCN-NEXT: v_dual_mov_b32 v43, s10 :: v_dual_mov_b32 v44, s11
-; GCN-NEXT: v_dual_mov_b32 v41, s8 :: v_dual_mov_b32 v42, s9
-; GCN-NEXT: v_readlane_b32 s47, v65, 11
-; GCN-NEXT: v_readlane_b32 s48, v65, 12
-; GCN-NEXT: v_readlane_b32 s49, v65, 13
-; GCN-NEXT: v_readlane_b32 s50, v65, 14
-; GCN-NEXT: v_readlane_b32 s51, v65, 15
-; GCN-NEXT: v_mov_b32_e32 v108, s17
+; GCN-NEXT: v_readlane_b32 s44, v139, 8
+; GCN-NEXT: v_readlane_b32 s45, v139, 9
+; GCN-NEXT: v_readlane_b32 s46, v139, 10
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[32:39], v[99:106], v[8:15], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v106, s23 :: v_dual_mov_b32 v103, s20
+; GCN-NEXT: v_dual_mov_b32 v105, s22 :: v_dual_mov_b32 v104, s21
+; GCN-NEXT: v_dual_mov_b32 v101, s18 :: v_dual_mov_b32 v102, s19
+; GCN-NEXT: v_mov_b32_e32 v99, s16
+; GCN-NEXT: v_readlane_b32 s47, v139, 11
+; GCN-NEXT: v_readlane_b32 s48, v139, 12
+; GCN-NEXT: v_readlane_b32 s49, v139, 13
+; GCN-NEXT: v_readlane_b32 s50, v139, 14
+; GCN-NEXT: v_readlane_b32 s51, v139, 15
+; GCN-NEXT: v_mov_b32_e32 v100, s17
; GCN-NEXT: s_load_b256 s[8:15], s[0:1], 0x120
; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0x120
; GCN-NEXT: s_load_b256 s[68:75], s[0:1], 0x140
; GCN-NEXT: s_load_b256 s[76:83], s[2:3], 0x140
; GCN-NEXT: s_load_b256 s[84:91], s[0:1], 0x160
; GCN-NEXT: s_load_b256 s[92:99], s[2:3], 0x160
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[99:106], v[49:56], v[82:89], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[74:81], v[115:122], v[57:64], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[82:89], v[41:48], v[107:114], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v41, s44 :: v_dual_mov_b32 v42, s45
-; GCN-NEXT: v_dual_mov_b32 v43, s46 :: v_dual_mov_b32 v44, s47
-; GCN-NEXT: v_dual_mov_b32 v45, s48 :: v_dual_mov_b32 v46, s49
-; GCN-NEXT: v_dual_mov_b32 v47, s50 :: v_dual_mov_b32 v48, s51
-; GCN-NEXT: v_readlane_b32 s44, v65, 16
-; GCN-NEXT: v_readlane_b32 s51, v65, 23
-; GCN-NEXT: v_readlane_b32 s45, v65, 17
-; GCN-NEXT: v_readlane_b32 s46, v65, 18
-; GCN-NEXT: v_readlane_b32 s47, v65, 19
-; GCN-NEXT: v_readlane_b32 s48, v65, 20
-; GCN-NEXT: v_readlane_b32 s49, v65, 21
-; GCN-NEXT: v_readlane_b32 s50, v65, 22
-; GCN-NEXT: v_mov_b32_e32 v114, s51
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[41:48], v[74:81], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[40:47], v[107:114], v[16:23], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[16:23], v[49:56], v[82:89], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[48:55], v[66:73], v[99:106], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v73, s51 :: v_dual_mov_b32 v72, s50
+; GCN-NEXT: v_dual_mov_b32 v71, s49 :: v_dual_mov_b32 v70, s48
+; GCN-NEXT: v_dual_mov_b32 v69, s47 :: v_dual_mov_b32 v68, s46
+; GCN-NEXT: v_dual_mov_b32 v67, s45 :: v_dual_mov_b32 v66, s44
+; GCN-NEXT: v_readlane_b32 s44, v139, 16
+; GCN-NEXT: v_readlane_b32 s51, v139, 23
+; GCN-NEXT: v_readlane_b32 s45, v139, 17
+; GCN-NEXT: v_readlane_b32 s46, v139, 18
+; GCN-NEXT: v_readlane_b32 s47, v139, 19
+; GCN-NEXT: v_readlane_b32 s48, v139, 20
+; GCN-NEXT: v_readlane_b32 s49, v139, 21
+; GCN-NEXT: v_readlane_b32 s50, v139, 22
+; GCN-NEXT: v_mov_b32_e32 v106, s51
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v56, s43 :: v_dual_mov_b32 v53, s40
-; GCN-NEXT: v_dual_mov_b32 v122, s31 :: v_dual_mov_b32 v119, s28
-; GCN-NEXT: v_dual_mov_b32 v108, s45 :: v_dual_mov_b32 v55, s42
-; GCN-NEXT: v_dual_mov_b32 v54, s41 :: v_dual_mov_b32 v51, s38
-; GCN-NEXT: v_dual_mov_b32 v52, s39 :: v_dual_mov_b32 v49, s36
-; GCN-NEXT: v_dual_mov_b32 v50, s37 :: v_dual_mov_b32 v121, s30
-; GCN-NEXT: v_dual_mov_b32 v120, s29 :: v_dual_mov_b32 v117, s26
-; GCN-NEXT: v_dual_mov_b32 v118, s27 :: v_dual_mov_b32 v115, s24
-; GCN-NEXT: v_mov_b32_e32 v116, s25
-; GCN-NEXT: v_dual_mov_b32 v64, s59 :: v_dual_mov_b32 v61, s56
-; GCN-NEXT: v_dual_mov_b32 v113, s50 :: v_dual_mov_b32 v112, s49
-; GCN-NEXT: v_mov_b32_e32 v109, s46
-; GCN-NEXT: v_dual_mov_b32 v111, s48 :: v_dual_mov_b32 v110, s47
-; GCN-NEXT: v_dual_mov_b32 v107, s44 :: v_dual_mov_b32 v58, s53
-; GCN-NEXT: v_dual_mov_b32 v63, s58 :: v_dual_mov_b32 v62, s57
-; GCN-NEXT: v_dual_mov_b32 v59, s54 :: v_dual_mov_b32 v60, s55
-; GCN-NEXT: v_dual_mov_b32 v57, s52 :: v_dual_mov_b32 v90, s60
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[16:23], v[49:56], v[115:122], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v122, s23 :: v_dual_mov_b32 v119, s20
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[8:15], v[41:48], v[107:114], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v48, s15 :: v_dual_mov_b32 v45, s12
-; GCN-NEXT: v_dual_mov_b32 v116, s17 :: v_dual_mov_b32 v49, s68
-; GCN-NEXT: v_mov_b32_e32 v47, s14
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[107:114], v[57:64], v[90:97], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v56, s75 :: v_dual_mov_b32 v97, s83
-; GCN-NEXT: v_dual_mov_b32 v90, s76 :: v_dual_mov_b32 v57, s84
-; GCN-NEXT: v_dual_mov_b32 v24, s92 :: v_dual_mov_b32 v27, s95
-; GCN-NEXT: v_dual_mov_b32 v46, s13 :: v_dual_mov_b32 v43, s10
-; GCN-NEXT: v_dual_mov_b32 v44, s11 :: v_dual_mov_b32 v41, s8
-; GCN-NEXT: v_dual_mov_b32 v42, s9 :: v_dual_mov_b32 v121, s22
-; GCN-NEXT: v_dual_mov_b32 v120, s21 :: v_dual_mov_b32 v117, s18
-; GCN-NEXT: v_dual_mov_b32 v118, s19 :: v_dual_mov_b32 v115, s16
-; GCN-NEXT: v_dual_mov_b32 v50, s69 :: v_dual_mov_b32 v51, s70
-; GCN-NEXT: v_dual_mov_b32 v52, s71 :: v_dual_mov_b32 v53, s72
-; GCN-NEXT: v_dual_mov_b32 v54, s73 :: v_dual_mov_b32 v55, s74
+; GCN-NEXT: v_dual_mov_b32 v100, s45 :: v_dual_mov_b32 v81, s43
+; GCN-NEXT: v_dual_mov_b32 v114, s31 :: v_dual_mov_b32 v111, s28
+; GCN-NEXT: v_dual_mov_b32 v105, s50 :: v_dual_mov_b32 v104, s49
+; GCN-NEXT: v_mov_b32_e32 v101, s46
+; GCN-NEXT: v_dual_mov_b32 v103, s48 :: v_dual_mov_b32 v102, s47
+; GCN-NEXT: v_dual_mov_b32 v99, s44 :: v_dual_mov_b32 v80, s42
+; GCN-NEXT: v_dual_mov_b32 v79, s41 :: v_dual_mov_b32 v78, s40
+; GCN-NEXT: v_dual_mov_b32 v77, s39 :: v_dual_mov_b32 v76, s38
+; GCN-NEXT: v_dual_mov_b32 v75, s37 :: v_dual_mov_b32 v74, s36
+; GCN-NEXT: v_dual_mov_b32 v113, s30 :: v_dual_mov_b32 v112, s29
+; GCN-NEXT: v_dual_mov_b32 v109, s26 :: v_dual_mov_b32 v110, s27
+; GCN-NEXT: v_dual_mov_b32 v107, s24 :: v_dual_mov_b32 v108, s25
+; GCN-NEXT: v_dual_mov_b32 v89, s59 :: v_dual_mov_b32 v88, s58
+; GCN-NEXT: v_dual_mov_b32 v87, s57 :: v_dual_mov_b32 v86, s56
+; GCN-NEXT: v_dual_mov_b32 v85, s55 :: v_dual_mov_b32 v84, s54
+; GCN-NEXT: v_dual_mov_b32 v83, s53 :: v_dual_mov_b32 v82, s52
+; GCN-NEXT: v_mov_b32_e32 v116, s61
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[56:63], v[66:73], v[99:106], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[64:71], v[74:81], v[107:114], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v106, s15 :: v_dual_mov_b32 v103, s12
+; GCN-NEXT: v_dual_mov_b32 v114, s75 :: v_dual_mov_b32 v111, s72
+; GCN-NEXT: v_mov_b32_e32 v105, s14
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[72:79], v[82:89], v[115:122], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v100, s9 :: v_dual_mov_b32 v89, s23
+; GCN-NEXT: v_dual_mov_b32 v108, s69 :: v_dual_mov_b32 v97, s83
+; GCN-NEXT: v_dual_mov_b32 v122, s91 :: v_dual_mov_b32 v119, s88
+; GCN-NEXT: v_dual_mov_b32 v130, s99 :: v_dual_mov_b32 v127, s96
+; GCN-NEXT: v_dual_mov_b32 v104, s13 :: v_dual_mov_b32 v101, s10
+; GCN-NEXT: v_dual_mov_b32 v102, s11 :: v_dual_mov_b32 v99, s8
+; GCN-NEXT: v_dual_mov_b32 v88, s22 :: v_dual_mov_b32 v87, s21
+; GCN-NEXT: v_dual_mov_b32 v86, s20 :: v_dual_mov_b32 v85, s19
+; GCN-NEXT: v_dual_mov_b32 v84, s18 :: v_dual_mov_b32 v83, s17
+; GCN-NEXT: v_dual_mov_b32 v82, s16 :: v_dual_mov_b32 v113, s74
+; GCN-NEXT: v_dual_mov_b32 v112, s73 :: v_dual_mov_b32 v109, s70
+; GCN-NEXT: v_dual_mov_b32 v110, s71 :: v_dual_mov_b32 v107, s68
; GCN-NEXT: v_dual_mov_b32 v96, s82 :: v_dual_mov_b32 v95, s81
; GCN-NEXT: v_dual_mov_b32 v94, s80 :: v_dual_mov_b32 v93, s79
; GCN-NEXT: v_dual_mov_b32 v92, s78 :: v_dual_mov_b32 v91, s77
-; GCN-NEXT: v_dual_mov_b32 v58, s85 :: v_dual_mov_b32 v59, s86
-; GCN-NEXT: v_dual_mov_b32 v60, s87 :: v_dual_mov_b32 v61, s88
-; GCN-NEXT: v_dual_mov_b32 v62, s89 :: v_dual_mov_b32 v63, s90
-; GCN-NEXT: v_dual_mov_b32 v64, s91 :: v_dual_mov_b32 v25, s93
-; GCN-NEXT: v_dual_mov_b32 v26, s94 :: v_dual_mov_b32 v29, s97
-; GCN-NEXT: v_dual_mov_b32 v28, s96 :: v_dual_mov_b32 v31, s99
-; GCN-NEXT: v_mov_b32_e32 v30, s98
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[131:138], v[41:48], v[115:122], v[0:7]
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[115:122], v[49:56], v[90:97], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v90, s76 :: v_dual_mov_b32 v121, s90
+; GCN-NEXT: v_dual_mov_b32 v120, s89 :: v_dual_mov_b32 v117, s86
+; GCN-NEXT: v_dual_mov_b32 v118, s87 :: v_dual_mov_b32 v115, s84
+; GCN-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v129, s98
+; GCN-NEXT: v_dual_mov_b32 v128, s97 :: v_dual_mov_b32 v125, s94
+; GCN-NEXT: v_dual_mov_b32 v126, s95 :: v_dual_mov_b32 v123, s92
+; GCN-NEXT: v_mov_b32_e32 v124, s93
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[131:138], v[99:106], v[82:89], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[80:87], v[107:114], v[90:97], v[0:7]
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[57:64], v[24:31], v[0:7]
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[0:7], v[115:122], v[123:130], v[0:7]
; GCN-NEXT: s_clause 0x17
-; GCN-NEXT: global_store_b128 v32, v[127:130], s[4:5] offset:16
-; GCN-NEXT: global_store_b128 v32, v[123:126], s[4:5]
-; GCN-NEXT: global_store_b128 v32, v[70:73], s[4:5] offset:48
-; GCN-NEXT: global_store_b128 v32, v[66:69], s[4:5] offset:32
-; GCN-NEXT: global_store_b128 v32, v[37:40], s[4:5] offset:80
-; GCN-NEXT: global_store_b128 v32, v[33:36], s[4:5] offset:64
-; GCN-NEXT: global_store_b128 v32, v[103:106], s[4:5] offset:112
-; GCN-NEXT: global_store_b128 v32, v[99:102], s[4:5] offset:96
-; GCN-NEXT: global_store_b128 v32, v[78:81], s[4:5] offset:144
-; GCN-NEXT: global_store_b128 v32, v[74:77], s[4:5] offset:128
-; GCN-NEXT: global_store_b128 v32, v[86:89], s[4:5] offset:176
-; GCN-NEXT: global_store_b128 v32, v[82:85], s[4:5] offset:160
-; GCN-NEXT: global_store_b128 v32, v[12:15], s[4:5] offset:208
-; GCN-NEXT: global_store_b128 v32, v[8:11], s[4:5] offset:192
-; GCN-NEXT: global_store_b128 v32, v[20:23], s[4:5] offset:240
-; GCN-NEXT: global_store_b128 v32, v[16:19], s[4:5] offset:224
-; GCN-NEXT: global_store_b128 v32, v[111:114], s[4:5] offset:272
-; GCN-NEXT: global_store_b128 v32, v[107:110], s[4:5] offset:256
-; GCN-NEXT: global_store_b128 v32, v[135:138], s[4:5] offset:304
-; GCN-NEXT: global_store_b128 v32, v[131:134], s[4:5] offset:288
-; GCN-NEXT: global_store_b128 v32, v[119:122], s[4:5] offset:336
-; GCN-NEXT: global_store_b128 v32, v[115:118], s[4:5] offset:320
-; GCN-NEXT: global_store_b128 v32, v[4:7], s[4:5] offset:368
-; GCN-NEXT: global_store_b128 v32, v[0:3], s[4:5] offset:352
+; GCN-NEXT: global_store_b128 v98, v[28:31], s[4:5] offset:16
+; GCN-NEXT: global_store_b128 v98, v[24:27], s[4:5]
+; GCN-NEXT: global_store_b128 v98, v[36:39], s[4:5] offset:48
+; GCN-NEXT: global_store_b128 v98, v[32:35], s[4:5] offset:32
+; GCN-NEXT: global_store_b128 v98, v[12:15], s[4:5] offset:80
+; GCN-NEXT: global_store_b128 v98, v[8:11], s[4:5] offset:64
+; GCN-NEXT: global_store_b128 v98, v[44:47], s[4:5] offset:112
+; GCN-NEXT: global_store_b128 v98, v[40:43], s[4:5] offset:96
+; GCN-NEXT: global_store_b128 v98, v[20:23], s[4:5] offset:144
+; GCN-NEXT: global_store_b128 v98, v[16:19], s[4:5] offset:128
+; GCN-NEXT: global_store_b128 v98, v[52:55], s[4:5] offset:176
+; GCN-NEXT: global_store_b128 v98, v[48:51], s[4:5] offset:160
+; GCN-NEXT: global_store_b128 v98, v[60:63], s[4:5] offset:208
+; GCN-NEXT: global_store_b128 v98, v[56:59], s[4:5] offset:192
+; GCN-NEXT: global_store_b128 v98, v[68:71], s[4:5] offset:240
+; GCN-NEXT: global_store_b128 v98, v[64:67], s[4:5] offset:224
+; GCN-NEXT: global_store_b128 v98, v[76:79], s[4:5] offset:272
+; GCN-NEXT: global_store_b128 v98, v[72:75], s[4:5] offset:256
+; GCN-NEXT: global_store_b128 v98, v[135:138], s[4:5] offset:304
+; GCN-NEXT: global_store_b128 v98, v[131:134], s[4:5] offset:288
+; GCN-NEXT: global_store_b128 v98, v[84:87], s[4:5] offset:336
+; GCN-NEXT: global_store_b128 v98, v[80:83], s[4:5] offset:320
+; GCN-NEXT: global_store_b128 v98, v[4:7], s[4:5] offset:368
+; GCN-NEXT: global_store_b128 v98, v[0:3], s[4:5] offset:352
; GCN-NEXT: s_nop 0
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll b/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
index 5ced1ec7f7ce2..0503fa6ae20a5 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma_multiple_32.ll
@@ -151,11 +151,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_unsigned(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_unsigned:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -171,11 +171,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_signed(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_signed:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[0,1,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -191,11 +191,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_unsigned(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_unsigned:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,0,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -211,11 +211,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_signed(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_signed:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0]
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0]
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,1,0]
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -231,11 +231,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_unsigned_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_unsigned_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -251,11 +251,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_unsigned_signed_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_unsigned_signed_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[0,1,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -271,11 +271,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_unsigned_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_unsigned_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,0,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
@@ -291,11 +291,11 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x16_ui8_signed_signed_clamp(<4 x i32> %A, <4 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W32-LABEL: test_wmma_i32_16x16x16_ui8_signed_signed_clamp:
; W32: ; %bb.0: ; %bb
-; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[27:34], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
+; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[20:27], v[0:3], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
; W32-NEXT: v_wmma_i32_16x16x16_iu8 v[8:15], v[4:7], v[4:7], v[8:15] neg_lo:[1,1,0] clamp
; W32-NEXT: s_clause 0x1
-; W32-NEXT: global_store_b128 v[16:17], v[31:34], off offset:16
-; W32-NEXT: global_store_b128 v[16:17], v[27:30], off
+; W32-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
+; W32-NEXT: global_store_b128 v[16:17], v[20:23], off
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[18:19], v[12:15], off offset:16
; W32-NEXT: global_store_b128 v[18:19], v[8:11], off
diff --git a/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll b/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
index cccdc88ab7aa4..138d80d16c52e 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma_multiple_64.ll
@@ -23,9 +23,9 @@ declare <4 x i32> @llvm.amdgcn.wmma.i32.16x16x16.iu4.v4i32.v2i32(i1 immarg, <2 x
define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<16 x half> %A, <16 x half> %B, <4 x float> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f32_16x16x16_f16:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f32_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f32_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f32_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -41,9 +41,9 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x16_bf16(<16 x i16> %A, <16 x i16> %B, <4 x float> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f32_16x16x16_bf16:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f32_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -59,9 +59,9 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_lo(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f16_16x16x16_f16_lo:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f16_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_f16_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -75,9 +75,9 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x16_f16_hi(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_f16_16x16x16_f16_hi:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_f16_16x16x16_f16 v[27:30], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
+; W64-NEXT: v_wmma_f16_16x16x16_f16 v[24:27], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[16:19], v[8:15], v[8:15], v[16:19] op_sel:[0,0,1]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -93,9 +93,9 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_lo(<16 x i16> %A, <16 x i16> %B, <8 x i16> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_bf16_16x16x16_bf16_lo:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19]
+; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19]
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
@@ -109,9 +109,9 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_hi(<16 x i16> %A, <16 x i16> %B, <8 x i16> %C, ptr addrspace(1) %out, ptr addrspace(1) %out2) {
; W64-LABEL: test_wmma_bf16_16x16x16_bf16_hi:
; W64: ; %bb.0: ; %bb
-; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[27:30], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
+; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[24:27], v[0:7], v[8:15], v[16:19] op_sel:[0,0,1]
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[16:19], v[8:15], v[8:15], v[16:19] op_sel:[0,0,1]
-; W64-NEXT: global_store_b128 v[20:21], v[27:30], off
+; W64-NEXT: global_store_b128 v[20:21], v[24:27], off
; W64-NEXT: global_store_b128 v[22:23], v[16:19], off
; W64-NEXT: s_endpgm
bb:
>From 6e01311be3ccb1b54b76413f19c4e218340e19ad Mon Sep 17 00:00:00 2001
From: shore <372660931 at qq.com>
Date: Wed, 1 Jul 2026 08:52:38 +0800
Subject: [PATCH 4/6] fix test case
---
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 56 ++++++++++----------
1 file changed, 28 insertions(+), 28 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 490f458168226..160f46daad9cb 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -32,21 +32,20 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_mov_b32_e32 v64, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
; COEXEC-NEXT: v_mov_b32_e32 v65, v0
-; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
-; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB0_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
-; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; COEXEC-NEXT: v_mov_b32_e32 v48, s2
; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
+; COEXEC-NEXT: s_and_b32 s3, s0, exec_lo
+; COEXEC-NEXT: s_cselect_b32 s3, 1, 0
; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v48 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[24:27], v48
; COEXEC-NEXT: ds_load_tr16_b128 v[28:31], v48 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v48 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[70:73], v48 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[66:69], v48 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v48 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v48 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v48 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v48 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v48 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[78:81], v48 offset:576
@@ -55,6 +54,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: ds_load_tr16_b128 v[82:85], v48 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[94:97], v48 offset:832
; COEXEC-NEXT: ds_load_tr16_b128 v[90:93], v48 offset:768
+; COEXEC-NEXT: s_cmp_lg_u32 s3, 1
; COEXEC-NEXT: s_wait_dscnt 0xc
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[24:31], v[16:23], v[58:65]
; COEXEC-NEXT: s_wait_dscnt 0x8
@@ -67,7 +67,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[32:39], v[66:73], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[74:81], v[40:47], v[50:57]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[90:97], v[82:89], v[0:7]
-; COEXEC-NEXT: s_cbranch_vccnz .LBB0_1
+; COEXEC-NEXT: s_cbranch_scc1 .LBB0_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v16, 0
@@ -102,23 +102,22 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
; GCN-NEXT: v_dual_mov_b32 v57, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v58, v0
-; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; GCN-NEXT: v_dual_mov_b32 v59, v0 :: v_dual_mov_b32 v60, v0
; GCN-NEXT: v_dual_mov_b32 v61, v0 :: v_dual_mov_b32 v62, v0
; GCN-NEXT: v_dual_mov_b32 v63, v0 :: v_dual_mov_b32 v64, v0
; GCN-NEXT: v_mov_b32_e32 v65, v0
+; GCN-NEXT: s_cselect_b32 s0, -1, 0
+; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: .LBB0_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mov_b32_e32 v48, s2
-; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s2, s2, s1
+; GCN-NEXT: s_and_b32 s3, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
; GCN-NEXT: ds_load_tr16_b128 v[16:19], v48
; GCN-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[24:27], v48 offset:128
@@ -135,6 +134,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:832
; GCN-NEXT: ds_load_tr16_b128 v[90:93], v48 offset:896
; GCN-NEXT: ds_load_tr16_b128 v[94:97], v48 offset:960
+; GCN-NEXT: s_cmp_lg_u32 s3, 1
; GCN-NEXT: s_wait_dscnt 0xc
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
; GCN-NEXT: s_wait_dscnt 0x8
@@ -147,7 +147,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
-; GCN-NEXT: s_cbranch_vccnz .LBB0_1
+; GCN-NEXT: s_cbranch_scc1 .LBB0_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
@@ -260,21 +260,21 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_dual_mov_b32 v91, v0 :: v_dual_mov_b32 v92, v0
; COEXEC-NEXT: v_dual_mov_b32 v93, v0 :: v_dual_mov_b32 v94, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
-; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
+; COEXEC-NEXT: s_bitcmp1_b32 s2, 0
; COEXEC-NEXT: v_mov_b32_e32 v95, v0
-; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
+; COEXEC-NEXT: s_cselect_b32 s2, -1, 0
; COEXEC-NEXT: v_mov_b32_e32 v96, v0
-; COEXEC-NEXT: s_xor_b32 s0, s0, -1
+; COEXEC-NEXT: s_xor_b32 s2, s2, -1
; COEXEC-NEXT: v_mov_b32_e32 v97, v0
-; COEXEC-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
-; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; COEXEC-NEXT: .LBB1_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
-; COEXEC-NEXT: s_add_co_i32 s7, s2, s6
-; COEXEC-NEXT: s_add_co_i32 s8, s3, s6
-; COEXEC-NEXT: s_add_co_i32 s6, s6, s1
+; COEXEC-NEXT: s_add_co_i32 s7, s0, s6
+; COEXEC-NEXT: s_add_co_i32 s8, s1, s6
+; COEXEC-NEXT: s_add_co_i32 s6, s6, s3
+; COEXEC-NEXT: s_and_b32 s9, s2, exec_lo
+; COEXEC-NEXT: s_cselect_b32 s9, 1, 0
; COEXEC-NEXT: v_mov_b32_e32 v80, s7
-; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
+; COEXEC-NEXT: s_cmp_lg_u32 s9, 1
; COEXEC-NEXT: v_mov_b32_e32 v81, s8
; COEXEC-NEXT: ds_load_tr16_b128 v[16:19], v80
; COEXEC-NEXT: ds_load_tr16_b128 v[20:23], v80 offset:64
@@ -332,7 +332,7 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[114:121], v[138:145], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[82:89], v[72:79], v[146:153], v[82:89]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[122:129], v[154:161], v[0:7]
-; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1
+; COEXEC-NEXT: s_cbranch_scc1 .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_mov_b32_e32 v16, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
@@ -367,25 +367,24 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
; GCN-NEXT: v_dual_mov_b32 v57, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v58, v0
-; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v16
; GCN-NEXT: v_dual_mov_b32 v59, v0 :: v_dual_mov_b32 v60, v0
; GCN-NEXT: v_dual_mov_b32 v61, v0 :: v_dual_mov_b32 v62, v0
; GCN-NEXT: v_dual_mov_b32 v63, v0 :: v_dual_mov_b32 v64, v0
; GCN-NEXT: v_mov_b32_e32 v65, v0
+; GCN-NEXT: s_cselect_b32 s0, -1, 0
+; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: .LBB1_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: s_add_co_i32 s7, s2, s6
; GCN-NEXT: s_add_co_i32 s8, s3, s6
; GCN-NEXT: v_dual_mov_b32 v48, s7 :: v_dual_mov_b32 v49, s8
-; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s6, s6, s1
+; GCN-NEXT: s_and_b32 s7, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s7, 1, 0
; GCN-NEXT: ds_load_tr16_b128 v[16:19], v48
; GCN-NEXT: ds_load_tr16_b128 v[20:23], v48 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[24:27], v49
@@ -402,6 +401,7 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: ds_load_tr16_b128 v[86:89], v48 offset:832
; GCN-NEXT: ds_load_tr16_b128 v[90:93], v49 offset:768
; GCN-NEXT: ds_load_tr16_b128 v[94:97], v49 offset:832
+; GCN-NEXT: s_cmp_lg_u32 s7, 1
; GCN-NEXT: s_wait_dscnt 0xc
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[58:65], v[16:23], v[24:31], v[58:65]
; GCN-NEXT: s_wait_dscnt 0x8
@@ -442,7 +442,7 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[66:73], v[32:39], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[50:57], v[40:47], v[74:81], v[50:57]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[82:89], v[90:97], v[0:7]
-; GCN-NEXT: s_cbranch_vccnz .LBB1_1
+; GCN-NEXT: s_cbranch_scc1 .LBB1_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
>From 978094f8cd4c6283b364d9faffd1da3935102362 Mon Sep 17 00:00:00 2001
From: shore <372660931 at qq.com>
Date: Thu, 9 Jul 2026 10:10:21 +0800
Subject: [PATCH 5/6] fix test case
---
.../llvm.amdgcn.sched.group.barrier.gfx11.ll | 104 +++++------
llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll | 174 +++++++++---------
2 files changed, 138 insertions(+), 140 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
index 6a4e62e9ee02c..4cc7b7485f6f9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx11.ll
@@ -12,43 +12,43 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_nc_u32_e32 v8, s0, v16
; GCN-NEXT: v_dual_mov_b32 v33, s1 :: v_dual_add_nc_u32 v16, s1, v16
+; GCN-NEXT: ds_load_b128 v[0:3], v8
; GCN-NEXT: ds_load_b128 v[4:7], v8 offset:16
; GCN-NEXT: ds_load_b128 v[21:24], v8 offset:2064
-; GCN-NEXT: ds_load_b128 v[38:41], v8 offset:6160
-; GCN-NEXT: ds_load_b128 v[55:58], v8 offset:12304
-; GCN-NEXT: ds_load_b128 v[12:15], v8 offset:20496
-; GCN-NEXT: ds_load_b128 v[0:3], v8
; GCN-NEXT: ds_load_b128 v[17:20], v8 offset:2048
+; GCN-NEXT: ds_load_b128 v[38:41], v8 offset:6160
; GCN-NEXT: ds_load_b128 v[34:37], v8 offset:6144
+; GCN-NEXT: ds_load_b128 v[55:58], v8 offset:12304
; GCN-NEXT: ds_load_b128 v[51:54], v8 offset:12288
+; GCN-NEXT: ds_load_b128 v[12:15], v8 offset:20496
; GCN-NEXT: ds_load_b128 v[8:11], v8 offset:20480
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)
-; GCN-NEXT: s_waitcnt lgkmcnt(3)
-; GCN-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v49, v24
-; GCN-NEXT: s_waitcnt lgkmcnt(2)
-; GCN-NEXT: v_mov_b32_e32 v66, v41
-; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mov_b32_e32 v74, v58
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v31, v6
+; GCN-NEXT: s_waitcnt lgkmcnt(8)
+; GCN-NEXT: v_mov_b32_e32 v28, v3
+; GCN-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v31, v6
; GCN-NEXT: v_dual_mov_b32 v30, v5 :: v_dual_mov_b32 v29, v4
-; GCN-NEXT: v_dual_mov_b32 v28, v3 :: v_dual_mov_b32 v27, v2
-; GCN-NEXT: v_dual_mov_b32 v26, v1 :: v_dual_mov_b32 v25, v0
-; GCN-NEXT: v_dual_mov_b32 v48, v23 :: v_dual_mov_b32 v47, v22
-; GCN-NEXT: v_dual_mov_b32 v46, v21 :: v_dual_mov_b32 v45, v20
-; GCN-NEXT: v_dual_mov_b32 v44, v19 :: v_dual_mov_b32 v43, v18
-; GCN-NEXT: v_dual_mov_b32 v42, v17 :: v_dual_mov_b32 v65, v40
+; GCN-NEXT: v_dual_mov_b32 v27, v2 :: v_dual_mov_b32 v26, v1
+; GCN-NEXT: v_mov_b32_e32 v25, v0
+; GCN-NEXT: s_waitcnt lgkmcnt(6)
+; GCN-NEXT: v_dual_mov_b32 v49, v24 :: v_dual_mov_b32 v48, v23
+; GCN-NEXT: v_dual_mov_b32 v47, v22 :: v_dual_mov_b32 v46, v21
+; GCN-NEXT: v_dual_mov_b32 v45, v20 :: v_dual_mov_b32 v44, v19
+; GCN-NEXT: v_dual_mov_b32 v43, v18 :: v_dual_mov_b32 v42, v17
+; GCN-NEXT: s_waitcnt lgkmcnt(4)
+; GCN-NEXT: v_dual_mov_b32 v66, v41 :: v_dual_mov_b32 v65, v40
; GCN-NEXT: v_dual_mov_b32 v64, v39 :: v_dual_mov_b32 v63, v38
; GCN-NEXT: v_dual_mov_b32 v62, v37 :: v_dual_mov_b32 v61, v36
; GCN-NEXT: v_dual_mov_b32 v60, v35 :: v_dual_mov_b32 v59, v34
-; GCN-NEXT: v_dual_mov_b32 v73, v57 :: v_dual_mov_b32 v72, v56
-; GCN-NEXT: v_dual_mov_b32 v71, v55 :: v_dual_mov_b32 v70, v54
-; GCN-NEXT: v_dual_mov_b32 v69, v53 :: v_dual_mov_b32 v68, v52
-; GCN-NEXT: v_mov_b32_e32 v67, v51
-; GCN-NEXT: v_dual_mov_b32 v81, v14 :: v_dual_mov_b32 v80, v13
-; GCN-NEXT: v_dual_mov_b32 v79, v12 :: v_dual_mov_b32 v78, v11
-; GCN-NEXT: v_dual_mov_b32 v77, v10 :: v_dual_mov_b32 v76, v9
-; GCN-NEXT: v_mov_b32_e32 v75, v8
+; GCN-NEXT: s_waitcnt lgkmcnt(2)
+; GCN-NEXT: v_dual_mov_b32 v74, v58 :: v_dual_mov_b32 v73, v57
+; GCN-NEXT: v_dual_mov_b32 v72, v56 :: v_dual_mov_b32 v71, v55
+; GCN-NEXT: v_dual_mov_b32 v70, v54 :: v_dual_mov_b32 v69, v53
+; GCN-NEXT: v_dual_mov_b32 v68, v52 :: v_dual_mov_b32 v67, v51
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v81, v14
+; GCN-NEXT: v_dual_mov_b32 v80, v13 :: v_dual_mov_b32 v79, v12
+; GCN-NEXT: v_dual_mov_b32 v78, v11 :: v_dual_mov_b32 v77, v10
+; GCN-NEXT: v_dual_mov_b32 v76, v9 :: v_dual_mov_b32 v75, v8
; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[25:32], v[0:7], v[0:7], v[25:32]
; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[42:49], v[17:24], v[17:24], v[42:49]
; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[59:66], v[34:41], v[34:41], v[59:66]
@@ -77,43 +77,43 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v8, s0, v16
; EXACTCUTOFF-NEXT: v_dual_mov_b32 v33, s1 :: v_dual_add_nc_u32 v16, s1, v16
+; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v8
; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v8 offset:16
; EXACTCUTOFF-NEXT: ds_load_b128 v[21:24], v8 offset:2064
-; EXACTCUTOFF-NEXT: ds_load_b128 v[38:41], v8 offset:6160
-; EXACTCUTOFF-NEXT: ds_load_b128 v[55:58], v8 offset:12304
-; EXACTCUTOFF-NEXT: ds_load_b128 v[12:15], v8 offset:20496
-; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v8
; EXACTCUTOFF-NEXT: ds_load_b128 v[17:20], v8 offset:2048
+; EXACTCUTOFF-NEXT: ds_load_b128 v[38:41], v8 offset:6160
; EXACTCUTOFF-NEXT: ds_load_b128 v[34:37], v8 offset:6144
+; EXACTCUTOFF-NEXT: ds_load_b128 v[55:58], v8 offset:12304
; EXACTCUTOFF-NEXT: ds_load_b128 v[51:54], v8 offset:12288
+; EXACTCUTOFF-NEXT: ds_load_b128 v[12:15], v8 offset:20496
; EXACTCUTOFF-NEXT: ds_load_b128 v[8:11], v8 offset:20480
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(3)
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v49, v24
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(2)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v66, v41
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v74, v58
-; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v31, v6
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(8)
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v28, v3
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v32, v7 :: v_dual_mov_b32 v31, v6
; EXACTCUTOFF-NEXT: v_dual_mov_b32 v30, v5 :: v_dual_mov_b32 v29, v4
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v28, v3 :: v_dual_mov_b32 v27, v2
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v26, v1 :: v_dual_mov_b32 v25, v0
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v48, v23 :: v_dual_mov_b32 v47, v22
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v46, v21 :: v_dual_mov_b32 v45, v20
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v44, v19 :: v_dual_mov_b32 v43, v18
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v42, v17 :: v_dual_mov_b32 v65, v40
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v27, v2 :: v_dual_mov_b32 v26, v1
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v25, v0
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(6)
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v49, v24 :: v_dual_mov_b32 v48, v23
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v47, v22 :: v_dual_mov_b32 v46, v21
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v45, v20 :: v_dual_mov_b32 v44, v19
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v43, v18 :: v_dual_mov_b32 v42, v17
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(4)
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v66, v41 :: v_dual_mov_b32 v65, v40
; EXACTCUTOFF-NEXT: v_dual_mov_b32 v64, v39 :: v_dual_mov_b32 v63, v38
; EXACTCUTOFF-NEXT: v_dual_mov_b32 v62, v37 :: v_dual_mov_b32 v61, v36
; EXACTCUTOFF-NEXT: v_dual_mov_b32 v60, v35 :: v_dual_mov_b32 v59, v34
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v73, v57 :: v_dual_mov_b32 v72, v56
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v71, v55 :: v_dual_mov_b32 v70, v54
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v69, v53 :: v_dual_mov_b32 v68, v52
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v67, v51
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v81, v14 :: v_dual_mov_b32 v80, v13
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v79, v12 :: v_dual_mov_b32 v78, v11
-; EXACTCUTOFF-NEXT: v_dual_mov_b32 v77, v10 :: v_dual_mov_b32 v76, v9
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v75, v8
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(2)
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v74, v58 :: v_dual_mov_b32 v73, v57
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v72, v56 :: v_dual_mov_b32 v71, v55
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v70, v54 :: v_dual_mov_b32 v69, v53
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v68, v52 :: v_dual_mov_b32 v67, v51
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v82, v15 :: v_dual_mov_b32 v81, v14
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v80, v13 :: v_dual_mov_b32 v79, v12
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v78, v11 :: v_dual_mov_b32 v77, v10
+; EXACTCUTOFF-NEXT: v_dual_mov_b32 v76, v9 :: v_dual_mov_b32 v75, v8
; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[25:32], v[0:7], v[0:7], v[25:32]
; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[42:49], v[17:24], v[17:24], v[42:49]
; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[59:66], v[34:41], v[34:41], v[59:66]
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
index bde3755c1862f..30baf7573f84f 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-bank-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GCN,GFX1100 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 < %s | FileCheck -check-prefix=GCN,GFX1151 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GCN,GFX1100 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1151 < %s | FileCheck -check-prefixes=GCN,GFX1151 %s
; Bank-conflict avoidance hint for v_wmma_* operands: the allocator should keep
; the WMMA source operands (and the accumulator/result) in distinct VGPR banks
@@ -198,43 +198,43 @@ define amdgpu_kernel void @wmma_f32_3x_high_pressure(ptr addrspace(1) %pa, ptr a
; GCN-NEXT: s_load_b256 s[24:31], s[46:47], 0x20
; GCN-NEXT: s_load_b256 s[36:43], s[44:45], 0x40
; GCN-NEXT: s_load_b256 s[44:51], s[46:47], 0x40
-; GCN-NEXT: s_mov_b32 s59, s52
; GCN-NEXT: s_mov_b32 s53, s52
; GCN-NEXT: s_mov_b32 s54, s52
; GCN-NEXT: s_mov_b32 s55, s52
; GCN-NEXT: s_mov_b32 s56, s52
; GCN-NEXT: s_mov_b32 s57, s52
; GCN-NEXT: s_mov_b32 s58, s52
+; GCN-NEXT: s_mov_b32 s59, s52
; GCN-NEXT: v_dual_mov_b32 v33, 0 :: v_dual_mov_b32 v0, s52
-; GCN-NEXT: v_mov_b32_e32 v7, s59
; GCN-NEXT: v_dual_mov_b32 v1, s53 :: v_dual_mov_b32 v2, s54
; GCN-NEXT: v_dual_mov_b32 v3, s55 :: v_dual_mov_b32 v4, s56
+; GCN-NEXT: v_dual_mov_b32 v5, s57 :: v_dual_mov_b32 v6, s58
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v41, s15
-; GCN-NEXT: v_dual_mov_b32 v38, s12 :: v_dual_mov_b32 v5, s57
-; GCN-NEXT: v_dual_mov_b32 v6, s58 :: v_dual_mov_b32 v23, s6
-; GCN-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v21, s4
-; GCN-NEXT: v_dual_mov_b32 v20, s3 :: v_dual_mov_b32 v19, s2
-; GCN-NEXT: v_dual_mov_b32 v18, s1 :: v_dual_mov_b32 v17, s0
-; GCN-NEXT: v_dual_mov_b32 v40, s14 :: v_dual_mov_b32 v39, s13
-; GCN-NEXT: v_dual_mov_b32 v36, s10 :: v_dual_mov_b32 v37, s11
-; GCN-NEXT: v_dual_mov_b32 v34, s8 :: v_dual_mov_b32 v35, s9
-; GCN-NEXT: v_dual_mov_b32 v58, s23 :: v_dual_mov_b32 v51, s16
-; GCN-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v57, s22
-; GCN-NEXT: v_dual_mov_b32 v56, s21 :: v_dual_mov_b32 v55, s20
-; GCN-NEXT: v_dual_mov_b32 v54, s19 :: v_dual_mov_b32 v53, s18
-; GCN-NEXT: v_dual_mov_b32 v52, s17 :: v_dual_mov_b32 v9, s25
-; GCN-NEXT: v_dual_mov_b32 v10, s26 :: v_dual_mov_b32 v11, s27
-; GCN-NEXT: v_dual_mov_b32 v12, s28 :: v_dual_mov_b32 v13, s29
-; GCN-NEXT: v_dual_mov_b32 v14, s30 :: v_dual_mov_b32 v15, s31
+; GCN-NEXT: v_dual_mov_b32 v7, s59 :: v_dual_mov_b32 v24, s7
+; GCN-NEXT: v_dual_mov_b32 v23, s6 :: v_dual_mov_b32 v22, s5
+; GCN-NEXT: v_dual_mov_b32 v21, s4 :: v_dual_mov_b32 v20, s3
+; GCN-NEXT: v_dual_mov_b32 v19, s2 :: v_dual_mov_b32 v18, s1
+; GCN-NEXT: v_dual_mov_b32 v17, s0 :: v_dual_mov_b32 v40, s14
+; GCN-NEXT: v_dual_mov_b32 v41, s15 :: v_dual_mov_b32 v38, s12
+; GCN-NEXT: v_dual_mov_b32 v39, s13 :: v_dual_mov_b32 v36, s10
+; GCN-NEXT: v_dual_mov_b32 v37, s11 :: v_dual_mov_b32 v34, s8
+; GCN-NEXT: v_dual_mov_b32 v35, s9 :: v_dual_mov_b32 v58, s23
+; GCN-NEXT: v_dual_mov_b32 v57, s22 :: v_dual_mov_b32 v56, s21
+; GCN-NEXT: v_dual_mov_b32 v55, s20 :: v_dual_mov_b32 v54, s19
+; GCN-NEXT: v_dual_mov_b32 v53, s18 :: v_dual_mov_b32 v52, s17
+; GCN-NEXT: v_dual_mov_b32 v51, s16 :: v_dual_mov_b32 v8, s24
+; GCN-NEXT: v_dual_mov_b32 v9, s25 :: v_dual_mov_b32 v10, s26
+; GCN-NEXT: v_dual_mov_b32 v11, s27 :: v_dual_mov_b32 v12, s28
+; GCN-NEXT: v_dual_mov_b32 v13, s29 :: v_dual_mov_b32 v14, s30
+; GCN-NEXT: v_dual_mov_b32 v15, s31 :: v_dual_mov_b32 v26, s37
; GCN-NEXT: v_dual_mov_b32 v25, s36 :: v_dual_mov_b32 v28, s39
+; GCN-NEXT: v_dual_mov_b32 v27, s38 :: v_dual_mov_b32 v30, s41
+; GCN-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v32, s43
; GCN-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v42, s44
-; GCN-NEXT: v_dual_mov_b32 v26, s37 :: v_dual_mov_b32 v27, s38
-; GCN-NEXT: v_dual_mov_b32 v30, s41 :: v_dual_mov_b32 v29, s40
-; GCN-NEXT: v_dual_mov_b32 v32, s43 :: v_dual_mov_b32 v43, s45
-; GCN-NEXT: v_dual_mov_b32 v44, s46 :: v_dual_mov_b32 v45, s47
-; GCN-NEXT: v_dual_mov_b32 v46, s48 :: v_dual_mov_b32 v47, s49
-; GCN-NEXT: v_dual_mov_b32 v48, s50 :: v_dual_mov_b32 v49, s51
+; GCN-NEXT: v_dual_mov_b32 v43, s45 :: v_dual_mov_b32 v44, s46
+; GCN-NEXT: v_dual_mov_b32 v45, s47 :: v_dual_mov_b32 v46, s48
+; GCN-NEXT: v_dual_mov_b32 v47, s49 :: v_dual_mov_b32 v48, s50
+; GCN-NEXT: v_mov_b32_e32 v49, s51
; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[59:66], v[17:24], v[34:41], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[16:23], v[51:58], v[8:15], v[0:7]
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_3)
@@ -288,8 +288,8 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: s_mov_b32 s12, s8
; GCN-NEXT: s_mov_b32 s13, s8
; GCN-NEXT: s_mov_b32 s15, s8
-; GCN-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
; GCN-NEXT: v_dual_mov_b32 v98, 0 :: v_dual_mov_b32 v1, s9
+; GCN-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
; GCN-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
; GCN-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -308,7 +308,7 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x40
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_dual_mov_b32 v114, s75 :: v_dual_mov_b32 v111, s72
-; GCN-NEXT: v_dual_mov_b32 v16, s76 :: v_dual_mov_b32 v19, s79
+; GCN-NEXT: v_dual_mov_b32 v112, s73 :: v_dual_mov_b32 v109, s70
; GCN-NEXT: v_writelane_b32 v139, s16, 0
; GCN-NEXT: v_dual_mov_b32 v40, s31 :: v_dual_mov_b32 v37, s28
; GCN-NEXT: v_dual_mov_b32 v39, s30 :: v_dual_mov_b32 v38, s29
@@ -318,90 +318,88 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: v_dual_mov_b32 v34, s25 :: v_dual_mov_b32 v73, s51
; GCN-NEXT: v_writelane_b32 v139, s18, 2
; GCN-NEXT: s_load_b256 s[24:31], s[0:1], 0xc0
-; GCN-NEXT: v_dual_mov_b32 v106, s43 :: v_dual_mov_b32 v103, s40
-; GCN-NEXT: v_dual_mov_b32 v66, s44 :: v_dual_mov_b32 v105, s42
-; GCN-NEXT: v_writelane_b32 v139, s19, 3
-; GCN-NEXT: v_dual_mov_b32 v104, s41 :: v_dual_mov_b32 v101, s38
-; GCN-NEXT: v_dual_mov_b32 v102, s39 :: v_dual_mov_b32 v99, s36
-; GCN-NEXT: v_writelane_b32 v139, s20, 4
; GCN-NEXT: v_dual_mov_b32 v72, s50 :: v_dual_mov_b32 v71, s49
; GCN-NEXT: v_dual_mov_b32 v70, s48 :: v_dual_mov_b32 v69, s47
-; GCN-NEXT: v_writelane_b32 v139, s21, 5
+; GCN-NEXT: v_writelane_b32 v139, s19, 3
; GCN-NEXT: v_dual_mov_b32 v68, s46 :: v_dual_mov_b32 v67, s45
-; GCN-NEXT: v_dual_mov_b32 v82, s92 :: v_dual_mov_b32 v85, s95
+; GCN-NEXT: v_dual_mov_b32 v66, s44 :: v_dual_mov_b32 v105, s42
+; GCN-NEXT: v_writelane_b32 v139, s20, 4
+; GCN-NEXT: v_dual_mov_b32 v106, s43 :: v_dual_mov_b32 v103, s40
+; GCN-NEXT: v_dual_mov_b32 v104, s41 :: v_dual_mov_b32 v101, s38
+; GCN-NEXT: v_writelane_b32 v139, s21, 5
+; GCN-NEXT: v_dual_mov_b32 v102, s39 :: v_dual_mov_b32 v99, s36
+; GCN-NEXT: v_dual_mov_b32 v42, s53 :: v_dual_mov_b32 v43, s54
; GCN-NEXT: v_writelane_b32 v139, s22, 6
-; GCN-NEXT: v_dual_mov_b32 v22, s82 :: v_dual_mov_b32 v49, s84
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[24:31], v[33:40], v[66:73], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v44, s55 :: v_dual_mov_b32 v45, s56
+; GCN-NEXT: v_dual_mov_b32 v46, s57 :: v_dual_mov_b32 v47, s58
; GCN-NEXT: v_writelane_b32 v139, s23, 7
; GCN-NEXT: s_load_b256 s[8:15], s[0:1], 0xa0
; GCN-NEXT: s_load_b256 s[16:23], s[2:3], 0xa0
-; GCN-NEXT: v_mov_b32_e32 v88, s98
-; GCN-NEXT: v_dual_mov_b32 v112, s73 :: v_dual_mov_b32 v109, s70
+; GCN-NEXT: v_dual_mov_b32 v48, s59 :: v_dual_mov_b32 v81, s67
+; GCN-NEXT: v_dual_mov_b32 v80, s66 :: v_dual_mov_b32 v79, s65
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_writelane_b32 v139, s24, 8
+; GCN-NEXT: v_dual_mov_b32 v78, s64 :: v_dual_mov_b32 v77, s63
+; GCN-NEXT: v_dual_mov_b32 v76, s62 :: v_dual_mov_b32 v75, s61
+; GCN-NEXT: v_writelane_b32 v139, s25, 9
+; GCN-NEXT: v_dual_mov_b32 v74, s60 :: v_dual_mov_b32 v113, s74
+; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x100
; GCN-NEXT: v_dual_mov_b32 v110, s71 :: v_dual_mov_b32 v107, s68
+; GCN-NEXT: v_writelane_b32 v139, s26, 10
; GCN-NEXT: v_dual_mov_b32 v108, s69 :: v_dual_mov_b32 v17, s77
-; GCN-NEXT: v_writelane_b32 v139, s25, 9
+; GCN-NEXT: v_dual_mov_b32 v16, s76 :: v_dual_mov_b32 v19, s79
+; GCN-NEXT: v_writelane_b32 v139, s27, 11
; GCN-NEXT: v_dual_mov_b32 v18, s78 :: v_dual_mov_b32 v21, s81
; GCN-NEXT: v_dual_mov_b32 v20, s80 :: v_dual_mov_b32 v23, s83
-; GCN-NEXT: v_writelane_b32 v139, s26, 10
-; GCN-NEXT: v_dual_mov_b32 v50, s85 :: v_dual_mov_b32 v51, s86
-; GCN-NEXT: v_dual_mov_b32 v52, s87 :: v_dual_mov_b32 v53, s88
-; GCN-NEXT: v_writelane_b32 v139, s27, 11
-; GCN-NEXT: v_dual_mov_b32 v73, s15 :: v_dual_mov_b32 v54, s89
-; GCN-NEXT: v_dual_mov_b32 v55, s90 :: v_dual_mov_b32 v56, s91
-; GCN-NEXT: v_mov_b32_e32 v83, s93
; GCN-NEXT: v_writelane_b32 v139, s28, 12
-; GCN-NEXT: v_dual_mov_b32 v84, s94 :: v_dual_mov_b32 v87, s97
-; GCN-NEXT: v_dual_mov_b32 v86, s96 :: v_dual_mov_b32 v89, s99
+; GCN-NEXT: v_dual_mov_b32 v22, s82 :: v_dual_mov_b32 v49, s84
+; GCN-NEXT: v_dual_mov_b32 v50, s85 :: v_dual_mov_b32 v51, s86
; GCN-NEXT: v_writelane_b32 v139, s29, 13
-; GCN-NEXT: v_dual_mov_b32 v72, s14 :: v_dual_mov_b32 v71, s13
-; GCN-NEXT: v_dual_mov_b32 v70, s12 :: v_dual_mov_b32 v69, s11
+; GCN-NEXT: v_dual_mov_b32 v52, s87 :: v_dual_mov_b32 v53, s88
+; GCN-NEXT: v_dual_mov_b32 v54, s89 :: v_dual_mov_b32 v55, s90
; GCN-NEXT: v_writelane_b32 v139, s30, 14
-; GCN-NEXT: v_dual_mov_b32 v68, s10 :: v_dual_mov_b32 v67, s9
-; GCN-NEXT: v_dual_mov_b32 v66, s8 :: v_dual_mov_b32 v41, s52
+; GCN-NEXT: v_dual_mov_b32 v56, s91 :: v_dual_mov_b32 v83, s93
+; GCN-NEXT: v_dual_mov_b32 v82, s92 :: v_dual_mov_b32 v85, s95
; GCN-NEXT: v_writelane_b32 v139, s31, 15
; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xc0
-; GCN-NEXT: v_dual_mov_b32 v48, s59 :: v_dual_mov_b32 v81, s67
-; GCN-NEXT: v_dual_mov_b32 v42, s53 :: v_dual_mov_b32 v43, s54
-; GCN-NEXT: v_dual_mov_b32 v44, s55 :: v_dual_mov_b32 v45, s56
-; GCN-NEXT: v_dual_mov_b32 v46, s57 :: v_dual_mov_b32 v47, s58
-; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x100
-; GCN-NEXT: v_dual_mov_b32 v80, s66 :: v_dual_mov_b32 v79, s65
-; GCN-NEXT: v_dual_mov_b32 v78, s64 :: v_dual_mov_b32 v77, s63
-; GCN-NEXT: v_dual_mov_b32 v76, s62 :: v_dual_mov_b32 v75, s61
-; GCN-NEXT: v_dual_mov_b32 v74, s60 :: v_dual_mov_b32 v113, s74
-; GCN-NEXT: s_load_b256 s[60:67], s[2:3], 0x100
+; GCN-NEXT: v_dual_mov_b32 v84, s94 :: v_dual_mov_b32 v87, s97
+; GCN-NEXT: v_dual_mov_b32 v86, s96 :: v_dual_mov_b32 v89, s99
+; GCN-NEXT: v_mov_b32_e32 v88, s98
+; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[24:31], v[33:40], v[66:73], v[0:7]
+; GCN-NEXT: v_dual_mov_b32 v73, s15 :: v_dual_mov_b32 v72, s14
+; GCN-NEXT: v_dual_mov_b32 v71, s13 :: v_dual_mov_b32 v70, s12
+; GCN-NEXT: v_dual_mov_b32 v69, s11 :: v_dual_mov_b32 v68, s10
+; GCN-NEXT: v_dual_mov_b32 v67, s9 :: v_dual_mov_b32 v66, s8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_dual_mov_b32 v122, s67 :: v_dual_mov_b32 v119, s64
+; GCN-NEXT: v_dual_mov_b32 v120, s65 :: v_dual_mov_b32 v117, s62
; GCN-NEXT: v_writelane_b32 v139, s24, 16
+; GCN-NEXT: v_dual_mov_b32 v118, s63 :: v_dual_mov_b32 v115, s60
+; GCN-NEXT: v_mov_b32_e32 v41, s52
+; GCN-NEXT: s_load_b256 s[52:59], s[0:1], 0x100
; GCN-NEXT: v_writelane_b32 v139, s25, 17
+; GCN-NEXT: v_mov_b32_e32 v121, s66
; GCN-NEXT: v_writelane_b32 v139, s26, 18
; GCN-NEXT: v_writelane_b32 v139, s27, 19
-; GCN-NEXT: v_dual_mov_b32 v122, s67 :: v_dual_mov_b32 v119, s64
; GCN-NEXT: v_writelane_b32 v139, s28, 20
-; GCN-NEXT: v_dual_mov_b32 v121, s66 :: v_dual_mov_b32 v120, s65
-; GCN-NEXT: v_dual_mov_b32 v117, s62 :: v_dual_mov_b32 v118, s63
-; GCN-NEXT: v_mov_b32_e32 v115, s60
; GCN-NEXT: v_writelane_b32 v139, s29, 21
; GCN-NEXT: v_writelane_b32 v139, s30, 22
; GCN-NEXT: v_writelane_b32 v139, s31, 23
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GCN-NEXT: v_readlane_b32 s24, v139, 0
-; GCN-NEXT: v_readlane_b32 s30, v139, 6
; GCN-NEXT: v_readlane_b32 s25, v139, 1
; GCN-NEXT: v_readlane_b32 s26, v139, 2
; GCN-NEXT: v_readlane_b32 s27, v139, 3
; GCN-NEXT: v_readlane_b32 s28, v139, 4
; GCN-NEXT: v_readlane_b32 s29, v139, 5
+; GCN-NEXT: v_readlane_b32 s30, v139, 6
; GCN-NEXT: v_readlane_b32 s31, v139, 7
-; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GCN-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v11, s27
-; GCN-NEXT: v_mov_b32_e32 v14, s30
; GCN-NEXT: v_dual_mov_b32 v100, s37 :: v_dual_mov_b32 v9, s25
; GCN-NEXT: s_load_b256 s[36:43], s[0:1], 0xe0
+; GCN-NEXT: v_dual_mov_b32 v8, s24 :: v_dual_mov_b32 v11, s27
; GCN-NEXT: v_dual_mov_b32 v10, s26 :: v_dual_mov_b32 v13, s29
; GCN-NEXT: v_dual_mov_b32 v12, s28 :: v_dual_mov_b32 v15, s31
+; GCN-NEXT: v_mov_b32_e32 v14, s30
; GCN-NEXT: s_load_b256 s[24:31], s[2:3], 0xe0
; GCN-NEXT: v_readlane_b32 s44, v139, 8
; GCN-NEXT: v_readlane_b32 s45, v139, 9
@@ -432,25 +430,25 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: v_dual_mov_b32 v69, s47 :: v_dual_mov_b32 v68, s46
; GCN-NEXT: v_dual_mov_b32 v67, s45 :: v_dual_mov_b32 v66, s44
; GCN-NEXT: v_readlane_b32 s44, v139, 16
-; GCN-NEXT: v_readlane_b32 s51, v139, 23
; GCN-NEXT: v_readlane_b32 s45, v139, 17
; GCN-NEXT: v_readlane_b32 s46, v139, 18
; GCN-NEXT: v_readlane_b32 s47, v139, 19
; GCN-NEXT: v_readlane_b32 s48, v139, 20
; GCN-NEXT: v_readlane_b32 s49, v139, 21
; GCN-NEXT: v_readlane_b32 s50, v139, 22
-; GCN-NEXT: v_mov_b32_e32 v106, s51
+; GCN-NEXT: v_readlane_b32 s51, v139, 23
+; GCN-NEXT: v_dual_mov_b32 v102, s47 :: v_dual_mov_b32 v99, s44
+; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GCN-NEXT: v_dual_mov_b32 v104, s49 :: v_dual_mov_b32 v101, s46
+; GCN-NEXT: v_dual_mov_b32 v106, s51 :: v_dual_mov_b32 v103, s48
+; GCN-NEXT: v_dual_mov_b32 v105, s50 :: v_dual_mov_b32 v100, s45
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_dual_mov_b32 v100, s45 :: v_dual_mov_b32 v81, s43
-; GCN-NEXT: v_dual_mov_b32 v114, s31 :: v_dual_mov_b32 v111, s28
-; GCN-NEXT: v_dual_mov_b32 v105, s50 :: v_dual_mov_b32 v104, s49
-; GCN-NEXT: v_mov_b32_e32 v101, s46
-; GCN-NEXT: v_dual_mov_b32 v103, s48 :: v_dual_mov_b32 v102, s47
-; GCN-NEXT: v_dual_mov_b32 v99, s44 :: v_dual_mov_b32 v80, s42
+; GCN-NEXT: v_dual_mov_b32 v81, s43 :: v_dual_mov_b32 v80, s42
; GCN-NEXT: v_dual_mov_b32 v79, s41 :: v_dual_mov_b32 v78, s40
; GCN-NEXT: v_dual_mov_b32 v77, s39 :: v_dual_mov_b32 v76, s38
; GCN-NEXT: v_dual_mov_b32 v75, s37 :: v_dual_mov_b32 v74, s36
-; GCN-NEXT: v_dual_mov_b32 v113, s30 :: v_dual_mov_b32 v112, s29
+; GCN-NEXT: v_dual_mov_b32 v113, s30 :: v_dual_mov_b32 v114, s31
+; GCN-NEXT: v_dual_mov_b32 v111, s28 :: v_dual_mov_b32 v112, s29
; GCN-NEXT: v_dual_mov_b32 v109, s26 :: v_dual_mov_b32 v110, s27
; GCN-NEXT: v_dual_mov_b32 v107, s24 :: v_dual_mov_b32 v108, s25
; GCN-NEXT: v_dual_mov_b32 v89, s59 :: v_dual_mov_b32 v88, s58
@@ -461,28 +459,28 @@ define amdgpu_kernel void @wmma_f32_12x_pressure(ptr addrspace(1) %pa, ptr addrs
; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[56:63], v[66:73], v[99:106], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[64:71], v[74:81], v[107:114], v[0:7]
; GCN-NEXT: v_dual_mov_b32 v106, s15 :: v_dual_mov_b32 v103, s12
-; GCN-NEXT: v_dual_mov_b32 v114, s75 :: v_dual_mov_b32 v111, s72
; GCN-NEXT: v_mov_b32_e32 v105, s14
; GCN-NEXT: v_wmma_f32_16x16x16_f16 v[72:79], v[82:89], v[115:122], v[0:7]
-; GCN-NEXT: v_dual_mov_b32 v100, s9 :: v_dual_mov_b32 v89, s23
-; GCN-NEXT: v_dual_mov_b32 v108, s69 :: v_dual_mov_b32 v97, s83
-; GCN-NEXT: v_dual_mov_b32 v122, s91 :: v_dual_mov_b32 v119, s88
-; GCN-NEXT: v_dual_mov_b32 v130, s99 :: v_dual_mov_b32 v127, s96
; GCN-NEXT: v_dual_mov_b32 v104, s13 :: v_dual_mov_b32 v101, s10
; GCN-NEXT: v_dual_mov_b32 v102, s11 :: v_dual_mov_b32 v99, s8
+; GCN-NEXT: v_dual_mov_b32 v100, s9 :: v_dual_mov_b32 v89, s23
; GCN-NEXT: v_dual_mov_b32 v88, s22 :: v_dual_mov_b32 v87, s21
; GCN-NEXT: v_dual_mov_b32 v86, s20 :: v_dual_mov_b32 v85, s19
; GCN-NEXT: v_dual_mov_b32 v84, s18 :: v_dual_mov_b32 v83, s17
; GCN-NEXT: v_dual_mov_b32 v82, s16 :: v_dual_mov_b32 v113, s74
+; GCN-NEXT: v_dual_mov_b32 v114, s75 :: v_dual_mov_b32 v111, s72
; GCN-NEXT: v_dual_mov_b32 v112, s73 :: v_dual_mov_b32 v109, s70
; GCN-NEXT: v_dual_mov_b32 v110, s71 :: v_dual_mov_b32 v107, s68
+; GCN-NEXT: v_dual_mov_b32 v108, s69 :: v_dual_mov_b32 v97, s83
; GCN-NEXT: v_dual_mov_b32 v96, s82 :: v_dual_mov_b32 v95, s81
; GCN-NEXT: v_dual_mov_b32 v94, s80 :: v_dual_mov_b32 v93, s79
; GCN-NEXT: v_dual_mov_b32 v92, s78 :: v_dual_mov_b32 v91, s77
; GCN-NEXT: v_dual_mov_b32 v90, s76 :: v_dual_mov_b32 v121, s90
+; GCN-NEXT: v_dual_mov_b32 v122, s91 :: v_dual_mov_b32 v119, s88
; GCN-NEXT: v_dual_mov_b32 v120, s89 :: v_dual_mov_b32 v117, s86
; GCN-NEXT: v_dual_mov_b32 v118, s87 :: v_dual_mov_b32 v115, s84
; GCN-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v129, s98
+; GCN-NEXT: v_dual_mov_b32 v130, s99 :: v_dual_mov_b32 v127, s96
; GCN-NEXT: v_dual_mov_b32 v128, s97 :: v_dual_mov_b32 v125, s94
; GCN-NEXT: v_dual_mov_b32 v126, s95 :: v_dual_mov_b32 v123, s92
; GCN-NEXT: v_mov_b32_e32 v124, s93
>From 5c65ee600099051b6829a91edeeaa13f36769387 Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Fri, 17 Jul 2026 16:31:47 +0800
Subject: [PATCH 6/6] remove unused variable & func
---
llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 7b5ab95f8e3ef..e70e16e96f1b1 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -618,13 +618,6 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
// load/store is enabled.
IndexedMap<uint32_t, VGPRBlock2IndexFunctor> MaskForVGPRBlockOps;
- // Maps each virtual register that is an operand of a bank-sensitive v_wmma_*
- // (matrix sources a multiple of 128 bits) to that instruction's other operand
- // registers. Populated once by GCNPreRAOptimizations and consumed by
- // SIRegisterInfo::getRegAllocationHints for VGPR bank-conflict avoidance, so
- // the hint hook does not rescan the function for every virtual register.
- DenseMap<Register, SmallVector<Register, 3>> WMMABankSiblings;
-
// Peak ArchVGPR pressure of the function (the "actual demand"), recorded by
// the machine scheduler (GCNScheduleDAGMILive) from the per-region pressure
// it already computes. Used by the WMMA bank hint to anchor its reserved bank
@@ -665,15 +658,6 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
return MaskForVGPRBlockOps.inBounds(RegisterBlock);
}
- // Accessors for the WMMA bank-hint map (see WMMABankSiblings).
- DenseMap<Register, SmallVector<Register, 3>> &getWMMABankSiblings() {
- return WMMABankSiblings;
- }
- const DenseMap<Register, SmallVector<Register, 3>> &
- getWMMABankSiblings() const {
- return WMMABankSiblings;
- }
-
unsigned getPeakVGPRPressure() const { return PeakVGPRPressure; }
void setPeakVGPRPressure(unsigned P) { PeakVGPRPressure = P; }
More information about the llvm-commits
mailing list