[llvm] [AMDGPU] Avoid wmma bank conflict for GFX11 and GFX12 (PR #205530)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 18:56:34 PDT 2026
================
@@ -4289,8 +4289,163 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
return false;
}
default:
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
- VRM);
+ break;
+ }
+
+ bool BaseImplRetVal =
+ TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, VRM);
+
+ // Append v_wmma_* bank-conflict avoidance candidates (gfx11/gfx12). This only
+ // augments the candidate order for VirtReg and never changes BaseImplRetVal.
+ if (ST.hasVGPRBankConflict())
+ addWMMABankConflictHints(VirtReg, Order, Hints, MF, VRM);
+
+ return BaseImplRetVal;
+}
+
+static void recordWMMABankSiblings(Register VirtReg, const MachineInstr &MI,
+ SmallVector<Register, 3> &Siblings,
+ const SIInstrInfo *TII,
+ const SIRegisterInfo *TRI,
+ const MachineRegisterInfo &MRI) {
+ // The consecutive-allocation VGPR bank collision only happens when each
+ // matrix operand is a multiple of 4 VGPRs (128 bits) wide; with narrower
+ // operands the start registers already fall on different banks. Restrict the
+ // hint to WMMAs whose matrix source operands are a multiple of 128 bits.
+ Register Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)->getReg();
+ Register Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)->getReg();
+ Register Vdst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
+ if (!TRI->isVGPR(MRI, Src0) || !TRI->isVGPR(MRI, Src1))
+ return;
+
+ auto saveSiblings = [&](SmallVector<Register, 3> Regs) {
+ if (!is_contained(Regs, VirtReg))
+ return;
+ for (Register Reg : Regs) {
+ if (Reg != VirtReg && !is_contained(Siblings, Reg))
+ Siblings.push_back(Reg);
+ }
+ };
+
+ // Src2 == Dst :=> 2 cycle penalty for bank convlict between src0, src1, src2
+ // Src2 != Dst :=> 1 cycle penalty for bank convlict between src0, src1
+ const MachineOperand *MOSrc2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+ if (MOSrc2 && MOSrc2->isReg() && MOSrc2->getReg().isVirtual() &&
+ MOSrc2->getReg() == Vdst) {
+ saveSiblings({Src0, Src1, Vdst});
+ return;
+ }
+
+ saveSiblings({Src0, Src1});
+}
+
+// VGPR bank-conflict avoidance for v_wmma_* operands on gfx11/gfx12.
+//
+// A WMMA reads src0/src1/src2 through per-bank read ports (bank = first VGPR
+// % 4); when two operands share a bank the instruction pays an extra
+// issue-latency cycle. GCNPreRAOptimizations records, for each operand of a
+// bank-sensitive WMMA, that instruction's other operands (WMMABankSiblings).
+// Here, for the siblings that already have a physreg, append the candidates
+// from a not-yet-used bank so the allocator prefers them.
+//
+// The bias is applied via the candidate list (not by recording an MRI
+// allocation hint) on purpose: an MRI hint feeds the allocation-priority
+// computation (VRM::hasKnownPreference) and can reorder allocation so the
+// result register displaces the live-in source operands, introducing copies.
+// Appending here happens after the copy hints emitted by the base
+// implementation, so coalescing keeps priority and only this register's
+// candidate order is affected (never the global allocation order).
+void SIRegisterInfo::addWMMABankConflictHints(Register VirtReg,
+ ArrayRef<MCPhysReg> Order,
+ SmallVectorImpl<MCPhysReg> &Hints,
+ const MachineFunction &MF,
+ const VirtRegMap *VRM) const {
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ if (!VRM || !isVGPR(MRI, VirtReg))
+ return;
+
+ const SIMachineFunctionInfo &FuncInfo = *MF.getInfo<SIMachineFunctionInfo>();
+ SmallVector<Register, 3> Siblings;
+ for (MachineInstr &MI : MRI.reg_nodbg_instructions(VirtReg)) {
+ if (!SIInstrInfo::isWMMA(MI))
+ continue;
+ recordWMMABankSiblings(VirtReg, MI, Siblings, TII, this, MRI);
+ }
+
+ if (Siblings.empty())
+ return;
+
+ // Banks already taken by allocated siblings.
+ unsigned UsedBankMask = 0;
+ for (Register Sibling : Siblings) {
+ Register Phys = Sibling;
+ if (Phys.isVirtual()) {
+ if (!VRM->hasPhys(Phys))
+ continue;
+ Phys = VRM->getPhys(Phys);
+ }
+ if (Phys.isPhysical() && isVGPR(MRI, Phys))
+ UsedBankMask |= 1u << (getHWRegIndex(Phys) % 4);
+ }
+
+ unsigned W = getRegSizeInBits(VirtReg, MRI) / 32;
+ if (W && (W % 4) != 0)
+ return;
+
+ // Build a hardware-index -> physreg map and record the highest start index.
+ // Order is not guaranteed to be sorted (non-kernel functions reorder their
+ // callee-saved VGPRs), so we look candidates up by index instead of trusting
+ // Order's sequence or its last element.
+ DenseMap<unsigned, MCPhysReg> IdxToReg;
+ unsigned MaxIdx = 0;
+ for (MCPhysReg PhysReg : Order) {
+ unsigned Idx = getHWRegIndex(PhysReg);
+ IdxToReg[Idx] = PhysReg;
+ MaxIdx = std::max(MaxIdx, Idx);
+ }
+
+ // Precompute once per function:
+ BitVector CalleeSavedUnits(getNumRegUnits());
+ for (const MCPhysReg *CSR = MRI.getCalleeSavedRegs(); CSR && *CSR; ++CSR)
+ for (MCRegUnit U : regunits(*CSR))
+ CalleeSavedUnits.set(static_cast<unsigned>(U));
+ // Then for a candidate physreg P (the tuple at Start):
+ auto isCalleeSaved = [&](MCRegister P) {
+ return any_of(regunits(P), [&](MCRegUnit U) {
+ return CalleeSavedUnits.test(static_cast<unsigned>(U));
+ });
+ };
+
+ // Anchor the candidate window to the function's actual VGPR footprint: peak
+ // WMMA-region pressure, rounded up to the allocation granule and capped by
+ // the VGPR budget. Without this the window is the top of the whole VGPR file
----------------
Shoreshen wrote:
Hi @shiltian , this is saying we cannot pushing the hint in order. For example we have 2 wmma:
1. wmma %a, %b, %c, %a
2. wmma %d, %e, %f, %d
If we pushing hint in order, we probably have:
1. %a = v[0:3], %b = v[5:8], %c = v[10:13]
2. %d = v[14:17], %e = v[19:22], %f = v[24:27]
This will create to many "hole"s in register (v4, v9, v18, v23) and increase the register allocation pressure. And with the increasing of number of wmma, the number of "hole"s will also increase.
Instead if we do:
1. %a = v[0:3], %b = v[9:12], %c = v[18:20]
2. %d = v[4:7], %e = v[13:16], %f = v[22:24]
This only have 2 holes (v8, v17), and the max vgpr index also reduced. The concept is separate vgpr into continues blocks that have same bank, for example:
Block_1: v[0:3], v[4:7];
Block_2: v[9:12], v[13:16];
Block_3: v[18:21], v[22:25];
Block_4: v[27:30], v[31:34];
The number of "hole"s is fixed at 3 and does not effect by the number of wmma instructions
https://github.com/llvm/llvm-project/pull/205530
More information about the llvm-commits
mailing list