[llvm] [AMDGPU] Break WAR Hazards Using Scavenged Registers to Assist Load Clustering (PR #216343)
Patrick Simmons via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 10:55:11 PDT 2026
https://github.com/linuxrocks123 updated https://github.com/llvm/llvm-project/pull/216343
>From b1ad496753b2516f0a9b88d9aacc52a20c715278 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Thu, 13 Aug 2026 10:00:16 -0500
Subject: [PATCH 01/10] Works, probably, maybe
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 10 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 312 ++++++++++++++++++
.../Target/AMDGPU/GCNBreakLoadClusterDeps.h | 23 ++
6 files changed, 350 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
create mode 100644 llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.h
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 9fe4123a27bbd..970d54b206434 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -589,6 +589,9 @@ void initializeAMDGPUSetWavePriorityLegacyPass(PassRegistry &);
void initializeGCNRewritePartialRegUsesLegacyPass(llvm::PassRegistry &);
extern char &GCNRewritePartialRegUsesID;
+void initializeGCNBreakLoadClusterDepsLegacyPass(llvm::PassRegistry &);
+extern char &GCNBreakLoadClusterDepsID;
+
void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
extern char &AMDGPUWaitSGPRHazardsLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 6da139ea0b59c..e6ba2131f048b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -118,6 +118,7 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
MACHINE_FUNCTION_PASS("amdgpu-asm-printer", AMDGPUAsmPrinterPass())
MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
+MACHINE_FUNCTION_PASS("amdgpu-break-load-cluster-deps", GCNBreakLoadClusterDepsPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 6f0b06eaf95bb..688166d03042b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -40,6 +40,7 @@
#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUUnifyDivergentExitNodes.h"
#include "AMDGPUWaitSGPRHazards.h"
+#include "GCNBreakLoadClusterDeps.h"
#include "GCNDPPCombine.h"
#include "GCNIterativeScheduler.h"
#include "GCNNSAReassign.h"
@@ -735,6 +736,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
initializeAMDGPUPrintfRuntimeBindingPass(*PR);
initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
+ initializeGCNBreakLoadClusterDepsLegacyPass(*PR);
initializeGCNNSAReassignLegacyPass(*PR);
initializeGCNPreRAOptimizationsLegacyPass(*PR);
initializeGCNPreRALongBranchRegLegacyPass(*PR);
@@ -1998,6 +2000,10 @@ void GCNPassConfig::addPostRegAlloc() {
}
void GCNPassConfig::addPreSched2() {
+ // Break false anti-dependencies on load-address chains before the post-RA
+ // scheduler so its load-clustering mutation can burst the loads.
+ if (TM->getOptLevel() > CodeGenOptLevel::None)
+ addPass(&GCNBreakLoadClusterDepsID);
if (TM->getOptLevel() > CodeGenOptLevel::None)
addPass(createSIShrinkInstructionsLegacyPass());
addPass(&SIPostRABundlerLegacyID);
@@ -2697,6 +2703,10 @@ void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) const {
}
void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) const {
+ // Break false anti-dependencies on load-address chains before the post-RA
+ // scheduler so its load-clustering mutation can burst the loads.
+ if (TM.getOptLevel() > CodeGenOptLevel::None)
+ addMachineFunctionPass(GCNBreakLoadClusterDepsPass(), PMW);
if (TM.getOptLevel() > CodeGenOptLevel::None)
addMachineFunctionPass(SIShrinkInstructionsPass(), PMW);
addMachineFunctionPass(SIPostRABundlerPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..c2f1a0c61b92f 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -125,6 +125,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUWaitSGPRHazards.cpp
AMDGPUUnifyDivergentExitNodes.cpp
R600MachineCFGStructurizer.cpp
+ GCNBreakLoadClusterDeps.cpp
GCNCreateVOPD.cpp
GCNDPPCombine.cpp
GCNHazardRecognizer.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
new file mode 100644
index 0000000000000..60a98a65c72bf
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -0,0 +1,312 @@
+//===- GCNBreakLoadClusterDeps.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// Post-RA pass that breaks false (WAR/WAW) anti-dependencies on the address
+/// computation feeding clusterable memory loads, so that the downstream post-RA
+/// load-clustering scheduler can issue the loads as a burst and expose more
+/// memory-level parallelism.
+///
+/// Register allocation may pack the per-lane index/address computation of
+/// several adjacent loads into a small set of registers (e.g. funneling every
+/// extracted index through a single scratch VGPR, or reusing one address
+/// register pair across two loads). Those reuses are anti-dependencies: they
+/// serialize the address chains and pin the loads apart even though the loads
+/// are semantically independent. The post-RA MachineScheduler's load-cluster
+/// mutation cannot rename registers, so it cannot undo them.
+///
+/// This pass renames the reused registers on those address chains to free
+/// registers scavenged from the function, bounded by the VGPR budget of the
+/// current occupancy so it never spends registers that would drop the number of
+/// concurrent waves. It performs no rescheduling itself: once the false
+/// dependencies are gone the existing load-cluster scheduler does the reorder.
+//===----------------------------------------------------------------------===//
+
+#include "GCNBreakLoadClusterDeps.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+
+#include <algorithm>
+#include <bitset>
+#include <unordered_set>
+#include <utility>
+#include <tuple>
+#include <vector>
+
+using std::bitset;
+using std::pair;
+using std::reverse;
+using std::tie;
+using std::unordered_set;
+using std::vector;
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-break-load-cluster-deps"
+
+namespace {
+
+/// Target-independent-of-pass-manager implementation.
+class GCNBreakLoadClusterDepsImpl {
+ const GCNSubtarget *ST = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+ const SIInstrInfo *TII = nullptr;
+ MachineRegisterInfo *MRI = nullptr;
+ unsigned occupancy_budget;
+
+ bitset<AMDGPU::NUM_TARGET_REGS> getVGPR32Lanes(Register Reg) const;
+ pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
+ get_uses_and_defs_for(MachineInstr &MI) const;
+
+public:
+ bool run(MachineFunction &MF);
+ bool runOnMachineBasicBlock(MachineBasicBlock &MBB);
+};
+
+/// Legacy-PM wrapper.
+class GCNBreakLoadClusterDepsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ GCNBreakLoadClusterDepsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override {
+ return "AMDGPU Break Load Cluster Dependencies";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // end anonymous namespace
+
+ // Append the 32-bit VGPR lanes of physical VGPR `Reg` (any width) to `Lanes`.
+bitset<AMDGPU::NUM_TARGET_REGS> GCNBreakLoadClusterDepsImpl::getVGPR32Lanes(Register Reg) const {
+ bitset<AMDGPU::NUM_TARGET_REGS> to_return;
+ assert(Reg.isPhysical());
+ const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
+ unsigned NumLanes = TRI->getRegSizeInBits(*RC).getFixedValue() / 32;
+ if (NumLanes <= 1) { // already a VGPR_32
+ to_return[Reg] = true;
+ return to_return;
+ }
+ for (unsigned C = 0; C < NumLanes; ++C)
+ to_return[TRI->getSubReg(Reg, TRI->getSubRegFromChannel(C))] = true;
+ return to_return;
+}
+
+pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
+GCNBreakLoadClusterDepsImpl::get_uses_and_defs_for(MachineInstr &MI) const {
+ pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
+ to_return;
+ for (unsigned i = 0; i < MI.getNumOperands(); i++)
+ if (MI.getOperand(i).isReg())
+ (*(MI.getOperand(i).isDef()
+ ? &to_return.first
+ : &to_return.second)) |= getVGPR32Lanes(MI.getOperand(i).getReg());
+ return to_return;
+}
+
+bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
+ MachineBasicBlock &MBB) {
+ bool to_return = false;
+
+ // Find clusterable loads whose address operands share a register with an
+ // earlier load's address, or whose address def chains funnel through a
+ // common scratch register (WAR/WAW anti-dependencies).
+ vector<MachineInstr *> all_vector_loads;
+ for (MachineInstr &MI : MBB)
+ if (MI.mayLoad() && MI.getOperand(0).isReg() && TRI->isVGPR(*MRI,MI.getOperand(0).getReg()))
+ all_vector_loads.push_back(&MI);
+
+ reverse(all_vector_loads.begin(), all_vector_loads.end()); // efficiency
+ bitset<AMDGPU::NUM_TARGET_REGS> used_load_source_physregs, used_load_dest_physregs;
+ while (!all_vector_loads.empty()) {
+ MachineInstr& vec_load_ins = *all_vector_loads.back();
+ bitset<AMDGPU::NUM_TARGET_REGS> ins_defs, ins_uses;
+ tie(ins_defs, ins_uses) = get_uses_and_defs_for(vec_load_ins);
+
+ // This means the load is not independent from previous loads
+ if ((ins_defs & used_load_dest_physregs).any()) {
+ used_load_source_physregs.reset();
+ used_load_dest_physregs.reset();
+ all_vector_loads.pop_back();
+ continue;
+ }
+
+ // Check if we have something to rename
+ bitset<AMDGPU::NUM_TARGET_REGS> war_conflicts =
+ ins_uses & used_load_source_physregs;
+ while (war_conflicts.any()) {
+ Register conflict_reg = war_conflicts._Find_first();
+ for (MachineBasicBlock::reverse_iterator
+ RIt = ++vec_load_ins.getReverseIterator(),
+ Top = MBB.rend();
+ RIt != Top; ++RIt)
+ if (RIt->definesRegister(conflict_reg, TRI)) {
+ // First, make sure we don't modify EXEC before redefining register.
+ bool exec_modified = false, redefined = false;
+ for (MachineBasicBlock::iterator It = std::next(RIt->getIterator());
+ It != MBB.end(); ++It)
+ if (It->definesRegister(AMDGPU::EXEC, TRI)) {
+ exec_modified = true;
+ break;
+ } else if (It->definesRegister(conflict_reg, TRI)) {
+ redefined = true;
+ break;
+ }
+
+ //Can't do anything if EXEC modified
+ if (exec_modified)
+ break;
+
+ LiveRegUnits LRU(*TRI);
+ LRU.addLiveOuts(MBB);
+
+ // If we're live out of the block and the conflicing reg hasn't been
+ // redefined, we can't do this with a block-local analysis.
+ if (!redefined && !LRU.available(conflict_reg))
+ break;
+
+ // Find the instruction which kills the def in RIt
+ MachineBasicBlock::iterator KillerIns = vec_load_ins;
+ for (MachineBasicBlock::iterator CandidateKiller =
+ std::next(KillerIns);
+ CandidateKiller != MBB.end(); ++CandidateKiller)
+ if (CandidateKiller->definesRegister(conflict_reg,TRI))
+ break;
+ else if (CandidateKiller->readsRegister(conflict_reg, TRI))
+ KillerIns = CandidateKiller;
+
+ // See what's free
+ for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin(); &*LiveRIt != &*KillerIns; ++LiveRIt)
+ LRU.stepBackward(*LiveRIt);
+ for (MachineBasicBlock::reverse_iterator AccumIt =
+ KillerIns->getReverseIterator();
+ AccumIt != MBB.rend(); ++AccumIt)
+ LRU.accumulate(*AccumIt);
+
+ // Iterate over registers in physical register class
+ const TargetRegisterClass &DefinedRegClass =
+ *TRI->getPhysRegBaseClass(RIt->getOperand(0).getReg());
+ unsigned i;
+ for (i = 0;
+ i < DefinedRegClass.getRegisters().size() &&
+ i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget;
+ i++)
+ if (LRU.available(DefinedRegClass.getRegisters()[i]))
+ break;
+
+ //Actually rename the register
+ if (i < DefinedRegClass.getRegisters().size() &&
+ i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget) {
+ for (unsigned op = 0; op < RIt->getNumOperands(); op++)
+ if (RIt->getOperand(op).isReg() && RIt->getOperand(op).isDef() &&
+ RIt->getOperand(op).getReg() == RIt->getOperand(0).getReg())
+ RIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+ for (MachineBasicBlock::iterator RenameIt =
+ std::next(RIt->getIterator());
+ RenameIt != KillerIns; ++RenameIt)
+ for (unsigned op = 0; op < RenameIt->getNumOperands(); op++)
+ if (RenameIt->getOperand(op).isReg() &&
+ RenameIt->getOperand(op).getReg() == RIt->getOperand(0).getReg())
+ RenameIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+ for (unsigned op = 0; op < KillerIns->getNumOperands(); op++)
+ if (KillerIns->getOperand(op).isReg() &&
+ KillerIns->getOperand(op).isUse() &&
+ KillerIns->getOperand(op).getReg() ==
+ RIt->getOperand(0).getReg())
+ KillerIns->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+
+ // Delete the conflict reg and all other conflicting registers we
+ // just handled
+ Register overlapping_reg = conflict_reg;
+ while (
+ TRI->regsOverlap(RIt->getOperand(0).getReg(), overlapping_reg))
+ war_conflicts[overlapping_reg++] = false;
+ }
+
+ // If we renamed, we're done. If we didn't rename, we can't get
+ // around this conflict, so we're also done.
+ break;
+ }
+
+ tie(ins_defs, ins_uses) = get_uses_and_defs_for(vec_load_ins);
+ war_conflicts[conflict_reg] = false;
+ }
+
+ // Coda
+ used_load_dest_physregs |= ins_defs;
+ used_load_source_physregs |= ins_uses;
+ all_vector_loads.pop_back();
+ }
+
+ // Scavenge free VGPRs and rename along each address def chain so the chains
+ // become register-disjoint, staying within the budget. The downstream
+ // post-RA load-cluster scheduler then reorders the now independent loads into
+ // a burst.
+
+ return to_return;
+}
+
+bool GCNBreakLoadClusterDepsImpl::run(MachineFunction &MF) {
+ ST = &MF.getSubtarget<GCNSubtarget>();
+ TRI = ST->getRegisterInfo();
+ TII = ST->getInstrInfo();
+ MRI = &MF.getRegInfo();
+ unsigned DynamicBlockSize =
+ MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled()
+ ? MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize()
+ : false;
+ occupancy_budget = ST->getMaxNumVGPRs(
+ ST->getOccupancyWithNumVGPRs(
+ TRI->getNumUsedPhysRegs(*MRI, AMDGPU::VGPR_32RegClass),
+ DynamicBlockSize),
+ DynamicBlockSize);
+
+ bool to_return = false;
+ for (MachineBasicBlock &MBB : MF)
+ to_return |= runOnMachineBasicBlock(MBB);
+
+ return false;
+}
+
+bool GCNBreakLoadClusterDepsLegacy::runOnMachineFunction(MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ return GCNBreakLoadClusterDepsImpl().run(MF);
+}
+
+PreservedAnalyses
+GCNBreakLoadClusterDepsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!GCNBreakLoadClusterDepsImpl().run(MF))
+ return PreservedAnalyses::all();
+
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+char GCNBreakLoadClusterDepsLegacy::ID = 0;
+
+char &llvm::GCNBreakLoadClusterDepsID = GCNBreakLoadClusterDepsLegacy::ID;
+
+INITIALIZE_PASS(GCNBreakLoadClusterDepsLegacy, DEBUG_TYPE,
+ "AMDGPU Break Load Cluster Dependencies", false, false)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.h b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.h
new file mode 100644
index 0000000000000..3a1e37f757202
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.h
@@ -0,0 +1,23 @@
+//===- GCNBreakLoadClusterDeps.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_GCNBREAKLOADCLUSTERDEPS_H
+#define LLVM_LIB_TARGET_AMDGPU_GCNBREAKLOADCLUSTERDEPS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class GCNBreakLoadClusterDepsPass
+ : public PassInfoMixin<GCNBreakLoadClusterDepsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_GCNBREAKLOADCLUSTERDEPS_H
>From 70f7678a45d50addca973974c89724044e6ef9e3 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Thu, 13 Aug 2026 10:14:44 -0500
Subject: [PATCH 02/10] Fixes
---
llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 60a98a65c72bf..faaf9822ed80c 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -213,32 +213,33 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
if (LRU.available(DefinedRegClass.getRegisters()[i]))
break;
- //Actually rename the register
+ // Actually rename the register
+ Register old_reg = RIt->getOperand(0).getReg();
if (i < DefinedRegClass.getRegisters().size() &&
i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget) {
for (unsigned op = 0; op < RIt->getNumOperands(); op++)
if (RIt->getOperand(op).isReg() && RIt->getOperand(op).isDef() &&
- RIt->getOperand(op).getReg() == RIt->getOperand(0).getReg())
+ RIt->getOperand(op).getReg() == old_reg)
RIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
for (MachineBasicBlock::iterator RenameIt =
std::next(RIt->getIterator());
RenameIt != KillerIns; ++RenameIt)
for (unsigned op = 0; op < RenameIt->getNumOperands(); op++)
if (RenameIt->getOperand(op).isReg() &&
- RenameIt->getOperand(op).getReg() == RIt->getOperand(0).getReg())
+ RenameIt->getOperand(op).getReg() == old_reg)
RenameIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
for (unsigned op = 0; op < KillerIns->getNumOperands(); op++)
if (KillerIns->getOperand(op).isReg() &&
KillerIns->getOperand(op).isUse() &&
KillerIns->getOperand(op).getReg() ==
- RIt->getOperand(0).getReg())
+ old_reg)
KillerIns->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
// Delete the conflict reg and all other conflicting registers we
// just handled
Register overlapping_reg = conflict_reg;
while (
- TRI->regsOverlap(RIt->getOperand(0).getReg(), overlapping_reg))
+ TRI->regsOverlap(old_reg, overlapping_reg))
war_conflicts[overlapping_reg++] = false;
}
>From c69e4686e4294e73c75a0b1275bdea98848c116d Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Thu, 13 Aug 2026 11:12:28 -0500
Subject: [PATCH 03/10] Getting there now
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 25 +++++++++++++------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index faaf9822ed80c..32f7441b66d3d 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -69,6 +69,7 @@ class GCNBreakLoadClusterDepsImpl {
bitset<AMDGPU::NUM_TARGET_REGS> getVGPR32Lanes(Register Reg) const;
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
get_uses_and_defs_for(MachineInstr &MI) const;
+ Register rename_register(Register from_reg, Register to_reg, Register rename_reg);
public:
bool run(MachineFunction &MF);
@@ -123,6 +124,15 @@ GCNBreakLoadClusterDepsImpl::get_uses_and_defs_for(MachineInstr &MI) const {
return to_return;
}
+Register GCNBreakLoadClusterDepsImpl::rename_register(Register from_reg, Register to_reg, Register rename_reg) {
+ if (rename_reg == from_reg)
+ return to_reg;
+ if (unsigned Idx =
+ TRI->getSubRegIndex(from_reg.asMCReg(), rename_reg.asMCReg()))
+ return TRI->getSubReg(to_reg.asMCReg(), Idx);
+ return rename_reg;
+}
+
bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
MachineBasicBlock &MBB) {
bool to_return = false;
@@ -219,21 +229,20 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget) {
for (unsigned op = 0; op < RIt->getNumOperands(); op++)
if (RIt->getOperand(op).isReg() && RIt->getOperand(op).isDef() &&
- RIt->getOperand(op).getReg() == old_reg)
- RIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+ TRI->regsOverlap(RIt->getOperand(op).getReg(),old_reg))
+ RIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RIt->getOperand(op).getReg()));
for (MachineBasicBlock::iterator RenameIt =
std::next(RIt->getIterator());
RenameIt != KillerIns; ++RenameIt)
for (unsigned op = 0; op < RenameIt->getNumOperands(); op++)
if (RenameIt->getOperand(op).isReg() &&
- RenameIt->getOperand(op).getReg() == old_reg)
- RenameIt->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+ TRI->regsOverlap(RenameIt->getOperand(op).getReg(),old_reg))
+ RenameIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RenameIt->getOperand(op).getReg()));
for (unsigned op = 0; op < KillerIns->getNumOperands(); op++)
if (KillerIns->getOperand(op).isReg() &&
KillerIns->getOperand(op).isUse() &&
- KillerIns->getOperand(op).getReg() ==
- old_reg)
- KillerIns->getOperand(op).setReg(DefinedRegClass.getRegisters()[i]);
+ TRI->regsOverlap(KillerIns->getOperand(op).getReg(),old_reg))
+ KillerIns->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],KillerIns->getOperand(op).getReg()));
// Delete the conflict reg and all other conflicting registers we
// just handled
@@ -285,7 +294,7 @@ bool GCNBreakLoadClusterDepsImpl::run(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF)
to_return |= runOnMachineBasicBlock(MBB);
- return false;
+ return to_return;
}
bool GCNBreakLoadClusterDepsLegacy::runOnMachineFunction(MachineFunction &MF) {
>From 528320846dca3d8a4f28fae6bef1b9e273d2a48f Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Thu, 13 Aug 2026 12:34:17 -0500
Subject: [PATCH 04/10] Cool, it works ... I think.
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 116 ++++++++++--------
1 file changed, 63 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 32f7441b66d3d..4bf1340a7f2d9 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -116,7 +116,7 @@ pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
GCNBreakLoadClusterDepsImpl::get_uses_and_defs_for(MachineInstr &MI) const {
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
to_return;
- for (unsigned i = 0; i < MI.getNumOperands(); i++)
+ for (unsigned i = 0; i < MI.getNumExplicitOperands(); i++)
if (MI.getOperand(i).isReg())
(*(MI.getOperand(i).isDef()
? &to_return.first
@@ -164,49 +164,67 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
bitset<AMDGPU::NUM_TARGET_REGS> war_conflicts =
ins_uses & used_load_source_physregs;
while (war_conflicts.any()) {
- Register conflict_reg = war_conflicts._Find_first();
- for (MachineBasicBlock::reverse_iterator
- RIt = ++vec_load_ins.getReverseIterator(),
- Top = MBB.rend();
- RIt != Top; ++RIt)
- if (RIt->definesRegister(conflict_reg, TRI)) {
- // First, make sure we don't modify EXEC before redefining register.
- bool exec_modified = false, redefined = false;
- for (MachineBasicBlock::iterator It = std::next(RIt->getIterator());
- It != MBB.end(); ++It)
- if (It->definesRegister(AMDGPU::EXEC, TRI)) {
- exec_modified = true;
- break;
- } else if (It->definesRegister(conflict_reg, TRI)) {
- redefined = true;
- break;
- }
-
- //Can't do anything if EXEC modified
- if (exec_modified)
- break;
-
- LiveRegUnits LRU(*TRI);
- LRU.addLiveOuts(MBB);
+ Register old_reg = war_conflicts._Find_first();
+ bitset<AMDGPU::NUM_TARGET_REGS> old_reg_war_conflicts;
+ for (const MachineOperand &operand : vec_load_ins.operands())
+ if (operand.isReg() && operand.isUse() &&
+ TRI->regsOverlap(old_reg, operand.getReg())) {
+ old_reg_war_conflicts |= getVGPR32Lanes(operand.getReg());
+ if (TRI->getPhysRegBaseClass(operand.getReg())->getSizeInBits() >
+ TRI->getPhysRegBaseClass(old_reg)->getSizeInBits())
+ old_reg = operand.getReg();
+ }
- // If we're live out of the block and the conflicing reg hasn't been
- // redefined, we can't do this with a block-local analysis.
- if (!redefined && !LRU.available(conflict_reg))
+ bitset<AMDGPU::NUM_TARGET_REGS> rit_reg_war_conflicts;
+ for (MachineBasicBlock::reverse_iterator RIt =
+ ++vec_load_ins.getReverseIterator();
+ RIt != MBB.rend(); ++RIt) {
+ if (RIt->modifiesRegister(old_reg, TRI))
+ rit_reg_war_conflicts |= get_uses_and_defs_for(*RIt).first;
+
+ if((old_reg_war_conflicts & ~rit_reg_war_conflicts).any())
+ continue;
+
+ // First, make sure we don't modify EXEC before redefining register.
+ bool exec_modified = false, redefined = false;
+ for (MachineBasicBlock::iterator It = std::next(RIt->getIterator());
+ It != MBB.end(); ++It)
+ if (It->definesRegister(AMDGPU::EXEC, TRI)) {
+ exec_modified = true;
break;
+ } else if (It->modifiesRegister(old_reg, TRI)) {
+ redefined = true;
+ }
+
+ //Can't do anything if EXEC modified
+ if (exec_modified)
+ break;
+
+ LiveRegUnits LRU(*TRI);
+ LRU.addLiveOuts(MBB);
+
+ // If we're live out of the block and the conflicing reg hasn't been
+ // redefined, we can't do this with a block-local analysis.
+ if (!redefined && !LRU.available(old_reg))
+ break;
- // Find the instruction which kills the def in RIt
- MachineBasicBlock::iterator KillerIns = vec_load_ins;
- for (MachineBasicBlock::iterator CandidateKiller =
- std::next(KillerIns);
- CandidateKiller != MBB.end(); ++CandidateKiller)
- if (CandidateKiller->definesRegister(conflict_reg,TRI))
+ // Find the instruction which kills the def in RIt
+ bitset<AMDGPU::NUM_TARGET_REGS> killed_subregs;
+ MachineBasicBlock::iterator KillerIns = vec_load_ins;
+ for (MachineBasicBlock::iterator CandidateKiller = std::next(KillerIns);
+ CandidateKiller != MBB.end(); ++CandidateKiller) {
+ if (CandidateKiller->modifiesRegister(old_reg, TRI)) {
+ killed_subregs |= get_uses_and_defs_for(*CandidateKiller).first;
+ if((rit_reg_war_conflicts & ~killed_subregs).none())
break;
- else if (CandidateKiller->readsRegister(conflict_reg, TRI))
- KillerIns = CandidateKiller;
-
- // See what's free
- for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin(); &*LiveRIt != &*KillerIns; ++LiveRIt)
- LRU.stepBackward(*LiveRIt);
+ }
+ if (CandidateKiller->readsRegister(old_reg, TRI))
+ KillerIns = CandidateKiller;
+ }
+
+ // See what's free
+ for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin(); &*LiveRIt != &*KillerIns; ++LiveRIt)
+ LRU.stepBackward(*LiveRIt);
for (MachineBasicBlock::reverse_iterator AccumIt =
KillerIns->getReverseIterator();
AccumIt != MBB.rend(); ++AccumIt)
@@ -214,7 +232,7 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
// Iterate over registers in physical register class
const TargetRegisterClass &DefinedRegClass =
- *TRI->getPhysRegBaseClass(RIt->getOperand(0).getReg());
+ *TRI->getPhysRegBaseClass(old_reg);
unsigned i;
for (i = 0;
i < DefinedRegClass.getRegisters().size() &&
@@ -224,32 +242,24 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
break;
// Actually rename the register
- Register old_reg = RIt->getOperand(0).getReg();
if (i < DefinedRegClass.getRegisters().size() &&
i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget) {
- for (unsigned op = 0; op < RIt->getNumOperands(); op++)
+ for (unsigned op = 0; op < RIt->getNumExplicitOperands(); op++)
if (RIt->getOperand(op).isReg() && RIt->getOperand(op).isDef() &&
TRI->regsOverlap(RIt->getOperand(op).getReg(),old_reg))
RIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RIt->getOperand(op).getReg()));
for (MachineBasicBlock::iterator RenameIt =
std::next(RIt->getIterator());
RenameIt != KillerIns; ++RenameIt)
- for (unsigned op = 0; op < RenameIt->getNumOperands(); op++)
+ for (unsigned op = 0; op < RenameIt->getNumExplicitOperands(); op++)
if (RenameIt->getOperand(op).isReg() &&
TRI->regsOverlap(RenameIt->getOperand(op).getReg(),old_reg))
RenameIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RenameIt->getOperand(op).getReg()));
- for (unsigned op = 0; op < KillerIns->getNumOperands(); op++)
+ for (unsigned op = 0; op < KillerIns->getNumExplicitOperands(); op++)
if (KillerIns->getOperand(op).isReg() &&
KillerIns->getOperand(op).isUse() &&
TRI->regsOverlap(KillerIns->getOperand(op).getReg(),old_reg))
KillerIns->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],KillerIns->getOperand(op).getReg()));
-
- // Delete the conflict reg and all other conflicting registers we
- // just handled
- Register overlapping_reg = conflict_reg;
- while (
- TRI->regsOverlap(old_reg, overlapping_reg))
- war_conflicts[overlapping_reg++] = false;
}
// If we renamed, we're done. If we didn't rename, we can't get
@@ -258,7 +268,7 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
}
tie(ins_defs, ins_uses) = get_uses_and_defs_for(vec_load_ins);
- war_conflicts[conflict_reg] = false;
+ war_conflicts &= ~getVGPR32Lanes(old_reg);
}
// Coda
>From ceee7057d4495993cefb57ab11454889547923f0 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Thu, 13 Aug 2026 13:09:41 -0500
Subject: [PATCH 05/10] Additional bugfix
---
llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 4bf1340a7f2d9..4437176cde154 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -100,7 +100,9 @@ class GCNBreakLoadClusterDepsLegacy : public MachineFunctionPass {
// Append the 32-bit VGPR lanes of physical VGPR `Reg` (any width) to `Lanes`.
bitset<AMDGPU::NUM_TARGET_REGS> GCNBreakLoadClusterDepsImpl::getVGPR32Lanes(Register Reg) const {
bitset<AMDGPU::NUM_TARGET_REGS> to_return;
- assert(Reg.isPhysical());
+ if (!TRI->isVGPR(*MRI,Reg))
+ return to_return;
+
const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
unsigned NumLanes = TRI->getRegSizeInBits(*RC).getFixedValue() / 32;
if (NumLanes <= 1) { // already a VGPR_32
@@ -211,7 +213,7 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
// Find the instruction which kills the def in RIt
bitset<AMDGPU::NUM_TARGET_REGS> killed_subregs;
MachineBasicBlock::iterator KillerIns = vec_load_ins;
- for (MachineBasicBlock::iterator CandidateKiller = std::next(KillerIns);
+ for (MachineBasicBlock::iterator CandidateKiller = KillerIns;
CandidateKiller != MBB.end(); ++CandidateKiller) {
if (CandidateKiller->modifiesRegister(old_reg, TRI)) {
killed_subregs |= get_uses_and_defs_for(*CandidateKiller).first;
>From 33eeb80d42beec5389b3d7f01d7000661149985f Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 14 Aug 2026 10:50:02 -0500
Subject: [PATCH 06/10] Cleanups and add testcase
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 271 ++++++++---------
.../CodeGen/AMDGPU/break-load-cluster-deps.ll | 276 ++++++++++++++++++
2 files changed, 416 insertions(+), 131 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/break-load-cluster-deps.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 4437176cde154..0f8f15e38fea5 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -40,9 +40,9 @@
#include <algorithm>
#include <bitset>
+#include <tuple>
#include <unordered_set>
#include <utility>
-#include <tuple>
#include <vector>
using std::bitset;
@@ -64,12 +64,12 @@ class GCNBreakLoadClusterDepsImpl {
const SIRegisterInfo *TRI = nullptr;
const SIInstrInfo *TII = nullptr;
MachineRegisterInfo *MRI = nullptr;
- unsigned occupancy_budget;
+ unsigned OccupancyBudget;
bitset<AMDGPU::NUM_TARGET_REGS> getVGPR32Lanes(Register Reg) const;
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
- get_uses_and_defs_for(MachineInstr &MI) const;
- Register rename_register(Register from_reg, Register to_reg, Register rename_reg);
+ getUsesAndDefsFor(MachineInstr &MI) const;
+ Register renameRegister(Register FromReg, Register ToReg, Register RenameReg);
public:
bool run(MachineFunction &MF);
@@ -97,194 +97,203 @@ class GCNBreakLoadClusterDepsLegacy : public MachineFunctionPass {
} // end anonymous namespace
- // Append the 32-bit VGPR lanes of physical VGPR `Reg` (any width) to `Lanes`.
-bitset<AMDGPU::NUM_TARGET_REGS> GCNBreakLoadClusterDepsImpl::getVGPR32Lanes(Register Reg) const {
- bitset<AMDGPU::NUM_TARGET_REGS> to_return;
- if (!TRI->isVGPR(*MRI,Reg))
- return to_return;
-
+// Append the 32-bit VGPR lanes of physical VGPR `Reg` (any width) to `Lanes`.
+bitset<AMDGPU::NUM_TARGET_REGS>
+GCNBreakLoadClusterDepsImpl::getVGPR32Lanes(Register Reg) const {
+ bitset<AMDGPU::NUM_TARGET_REGS> ToReturn;
+ if (!TRI->isVGPR(*MRI, Reg))
+ return ToReturn;
+
const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
unsigned NumLanes = TRI->getRegSizeInBits(*RC).getFixedValue() / 32;
if (NumLanes <= 1) { // already a VGPR_32
- to_return[Reg] = true;
- return to_return;
+ ToReturn[Reg] = true;
+ return ToReturn;
}
for (unsigned C = 0; C < NumLanes; ++C)
- to_return[TRI->getSubReg(Reg, TRI->getSubRegFromChannel(C))] = true;
- return to_return;
+ ToReturn[TRI->getSubReg(Reg, TRI->getSubRegFromChannel(C))] = true;
+ return ToReturn;
}
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
-GCNBreakLoadClusterDepsImpl::get_uses_and_defs_for(MachineInstr &MI) const {
+GCNBreakLoadClusterDepsImpl::getUsesAndDefsFor(MachineInstr &MI) const {
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
- to_return;
- for (unsigned i = 0; i < MI.getNumExplicitOperands(); i++)
- if (MI.getOperand(i).isReg())
- (*(MI.getOperand(i).isDef()
- ? &to_return.first
- : &to_return.second)) |= getVGPR32Lanes(MI.getOperand(i).getReg());
- return to_return;
+ ToReturn;
+ for (unsigned I = 0; I < MI.getNumExplicitOperands(); I++)
+ if (MI.getOperand(I).isReg())
+ (*(MI.getOperand(I).isDef() ? &ToReturn.first : &ToReturn.second)) |=
+ getVGPR32Lanes(MI.getOperand(I).getReg());
+ return ToReturn;
}
-Register GCNBreakLoadClusterDepsImpl::rename_register(Register from_reg, Register to_reg, Register rename_reg) {
- if (rename_reg == from_reg)
- return to_reg;
+Register GCNBreakLoadClusterDepsImpl::renameRegister(Register FromReg,
+ Register ToReg,
+ Register RenameReg) {
+ if (RenameReg == FromReg)
+ return ToReg;
if (unsigned Idx =
- TRI->getSubRegIndex(from_reg.asMCReg(), rename_reg.asMCReg()))
- return TRI->getSubReg(to_reg.asMCReg(), Idx);
- return rename_reg;
+ TRI->getSubRegIndex(FromReg.asMCReg(), RenameReg.asMCReg()))
+ return TRI->getSubReg(ToReg.asMCReg(), Idx);
+ return RenameReg;
}
bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
MachineBasicBlock &MBB) {
- bool to_return = false;
+ bool ToReturn = false;
// Find clusterable loads whose address operands share a register with an
// earlier load's address, or whose address def chains funnel through a
// common scratch register (WAR/WAW anti-dependencies).
- vector<MachineInstr *> all_vector_loads;
+ vector<MachineInstr *> AllVectorLoads;
for (MachineInstr &MI : MBB)
- if (MI.mayLoad() && MI.getOperand(0).isReg() && TRI->isVGPR(*MRI,MI.getOperand(0).getReg()))
- all_vector_loads.push_back(&MI);
-
- reverse(all_vector_loads.begin(), all_vector_loads.end()); // efficiency
- bitset<AMDGPU::NUM_TARGET_REGS> used_load_source_physregs, used_load_dest_physregs;
- while (!all_vector_loads.empty()) {
- MachineInstr& vec_load_ins = *all_vector_loads.back();
- bitset<AMDGPU::NUM_TARGET_REGS> ins_defs, ins_uses;
- tie(ins_defs, ins_uses) = get_uses_and_defs_for(vec_load_ins);
-
+ if (MI.mayLoad() && MI.getOperand(0).isReg() &&
+ TRI->isVGPR(*MRI, MI.getOperand(0).getReg()))
+ AllVectorLoads.push_back(&MI);
+
+ reverse(AllVectorLoads.begin(), AllVectorLoads.end()); // efficiency
+ bitset<AMDGPU::NUM_TARGET_REGS> UsedLoadSourcePhysregs, UsedLoadDestPhysregs;
+ while (!AllVectorLoads.empty()) {
+ MachineInstr &VecLoadIns = *AllVectorLoads.back();
+ bitset<AMDGPU::NUM_TARGET_REGS> InsDefs, InsUses;
+ tie(InsDefs, InsUses) = getUsesAndDefsFor(VecLoadIns);
+
// This means the load is not independent from previous loads
- if ((ins_defs & used_load_dest_physregs).any()) {
- used_load_source_physregs.reset();
- used_load_dest_physregs.reset();
- all_vector_loads.pop_back();
+ if ((InsDefs & UsedLoadDestPhysregs).any()) {
+ UsedLoadSourcePhysregs.reset();
+ UsedLoadDestPhysregs.reset();
+ AllVectorLoads.pop_back();
continue;
}
// Check if we have something to rename
- bitset<AMDGPU::NUM_TARGET_REGS> war_conflicts =
- ins_uses & used_load_source_physregs;
- while (war_conflicts.any()) {
- Register old_reg = war_conflicts._Find_first();
- bitset<AMDGPU::NUM_TARGET_REGS> old_reg_war_conflicts;
- for (const MachineOperand &operand : vec_load_ins.operands())
- if (operand.isReg() && operand.isUse() &&
- TRI->regsOverlap(old_reg, operand.getReg())) {
- old_reg_war_conflicts |= getVGPR32Lanes(operand.getReg());
- if (TRI->getPhysRegBaseClass(operand.getReg())->getSizeInBits() >
- TRI->getPhysRegBaseClass(old_reg)->getSizeInBits())
- old_reg = operand.getReg();
+ bitset<AMDGPU::NUM_TARGET_REGS> WarConflicts =
+ InsUses & UsedLoadSourcePhysregs;
+ while (WarConflicts.any()) {
+ Register OldReg = WarConflicts._Find_first();
+ bitset<AMDGPU::NUM_TARGET_REGS> OldRegWarConflicts;
+ for (const MachineOperand &Operand : VecLoadIns.operands())
+ if (Operand.isReg() && Operand.isUse() &&
+ TRI->regsOverlap(OldReg, Operand.getReg())) {
+ OldRegWarConflicts |= getVGPR32Lanes(Operand.getReg());
+ if (TRI->getPhysRegBaseClass(Operand.getReg())->getSizeInBits() >
+ TRI->getPhysRegBaseClass(OldReg)->getSizeInBits())
+ OldReg = Operand.getReg();
}
- bitset<AMDGPU::NUM_TARGET_REGS> rit_reg_war_conflicts;
+ bitset<AMDGPU::NUM_TARGET_REGS> RitRegWarConflicts;
for (MachineBasicBlock::reverse_iterator RIt =
- ++vec_load_ins.getReverseIterator();
+ ++VecLoadIns.getReverseIterator();
RIt != MBB.rend(); ++RIt) {
- if (RIt->modifiesRegister(old_reg, TRI))
- rit_reg_war_conflicts |= get_uses_and_defs_for(*RIt).first;
+ if (RIt->modifiesRegister(OldReg, TRI))
+ RitRegWarConflicts |= getUsesAndDefsFor(*RIt).first;
- if((old_reg_war_conflicts & ~rit_reg_war_conflicts).any())
+ if ((OldRegWarConflicts & ~RitRegWarConflicts).any())
continue;
-
+
// First, make sure we don't modify EXEC before redefining register.
- bool exec_modified = false, redefined = false;
+ bool ExecModified = false, Redefined = false;
for (MachineBasicBlock::iterator It = std::next(RIt->getIterator());
It != MBB.end(); ++It)
if (It->definesRegister(AMDGPU::EXEC, TRI)) {
- exec_modified = true;
+ ExecModified = true;
break;
- } else if (It->modifiesRegister(old_reg, TRI)) {
- redefined = true;
+ } else if (It->modifiesRegister(OldReg, TRI)) {
+ Redefined = true;
}
-
- //Can't do anything if EXEC modified
- if (exec_modified)
+
+ // Can't do anything if EXEC modified
+ if (ExecModified)
break;
-
+
LiveRegUnits LRU(*TRI);
LRU.addLiveOuts(MBB);
-
+
// If we're live out of the block and the conflicing reg hasn't been
// redefined, we can't do this with a block-local analysis.
- if (!redefined && !LRU.available(old_reg))
+ if (!Redefined && !LRU.available(OldReg))
break;
// Find the instruction which kills the def in RIt
- bitset<AMDGPU::NUM_TARGET_REGS> killed_subregs;
- MachineBasicBlock::iterator KillerIns = vec_load_ins;
+ bitset<AMDGPU::NUM_TARGET_REGS> KilledSubregs;
+ MachineBasicBlock::iterator KillerIns = VecLoadIns;
for (MachineBasicBlock::iterator CandidateKiller = KillerIns;
CandidateKiller != MBB.end(); ++CandidateKiller) {
- if (CandidateKiller->modifiesRegister(old_reg, TRI)) {
- killed_subregs |= get_uses_and_defs_for(*CandidateKiller).first;
- if((rit_reg_war_conflicts & ~killed_subregs).none())
+ if (CandidateKiller->modifiesRegister(OldReg, TRI)) {
+ KilledSubregs |= getUsesAndDefsFor(*CandidateKiller).first;
+ if ((RitRegWarConflicts & ~KilledSubregs).none())
break;
}
- if (CandidateKiller->readsRegister(old_reg, TRI))
+ if (CandidateKiller->readsRegister(OldReg, TRI))
KillerIns = CandidateKiller;
}
-
+
// See what's free
- for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin(); &*LiveRIt != &*KillerIns; ++LiveRIt)
+ for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin();
+ &*LiveRIt != &*KillerIns; ++LiveRIt)
LRU.stepBackward(*LiveRIt);
- for (MachineBasicBlock::reverse_iterator AccumIt =
- KillerIns->getReverseIterator();
- AccumIt != MBB.rend(); ++AccumIt)
- LRU.accumulate(*AccumIt);
-
- // Iterate over registers in physical register class
- const TargetRegisterClass &DefinedRegClass =
- *TRI->getPhysRegBaseClass(old_reg);
- unsigned i;
- for (i = 0;
- i < DefinedRegClass.getRegisters().size() &&
- i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget;
- i++)
- if (LRU.available(DefinedRegClass.getRegisters()[i]))
- break;
-
- // Actually rename the register
- if (i < DefinedRegClass.getRegisters().size() &&
- i * DefinedRegClass.getSizeInBits() / 32 < occupancy_budget) {
- for (unsigned op = 0; op < RIt->getNumExplicitOperands(); op++)
- if (RIt->getOperand(op).isReg() && RIt->getOperand(op).isDef() &&
- TRI->regsOverlap(RIt->getOperand(op).getReg(),old_reg))
- RIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RIt->getOperand(op).getReg()));
- for (MachineBasicBlock::iterator RenameIt =
- std::next(RIt->getIterator());
- RenameIt != KillerIns; ++RenameIt)
- for (unsigned op = 0; op < RenameIt->getNumExplicitOperands(); op++)
- if (RenameIt->getOperand(op).isReg() &&
- TRI->regsOverlap(RenameIt->getOperand(op).getReg(),old_reg))
- RenameIt->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],RenameIt->getOperand(op).getReg()));
- for (unsigned op = 0; op < KillerIns->getNumExplicitOperands(); op++)
- if (KillerIns->getOperand(op).isReg() &&
- KillerIns->getOperand(op).isUse() &&
- TRI->regsOverlap(KillerIns->getOperand(op).getReg(),old_reg))
- KillerIns->getOperand(op).setReg(rename_register(old_reg,DefinedRegClass.getRegisters()[i],KillerIns->getOperand(op).getReg()));
- }
+ for (MachineBasicBlock::reverse_iterator AccumIt =
+ KillerIns->getReverseIterator();
+ AccumIt != MBB.rend(); ++AccumIt)
+ LRU.accumulate(*AccumIt);
+
+ // Iterate over registers in physical register class
+ const TargetRegisterClass &DefinedRegClass =
+ *TRI->getPhysRegBaseClass(OldReg);
+ unsigned I;
+ for (I = 0; I < DefinedRegClass.getRegisters().size() &&
+ I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget;
+ I++)
+ if (LRU.available(DefinedRegClass.getRegisters()[I]))
+ break;
- // If we renamed, we're done. If we didn't rename, we can't get
- // around this conflict, so we're also done.
- break;
+ // Actually rename the register
+ if (I < DefinedRegClass.getRegisters().size() &&
+ I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget) {
+ for (unsigned Op = 0; Op < RIt->getNumExplicitOperands(); Op++)
+ if (RIt->getOperand(Op).isReg() && RIt->getOperand(Op).isDef() &&
+ TRI->regsOverlap(RIt->getOperand(Op).getReg(), OldReg))
+ RIt->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ RIt->getOperand(Op).getReg()));
+ for (MachineBasicBlock::iterator RenameIt =
+ std::next(RIt->getIterator());
+ RenameIt != KillerIns; ++RenameIt)
+ for (unsigned Op = 0; Op < RenameIt->getNumExplicitOperands(); Op++)
+ if (RenameIt->getOperand(Op).isReg() &&
+ TRI->regsOverlap(RenameIt->getOperand(Op).getReg(), OldReg))
+ RenameIt->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ RenameIt->getOperand(Op).getReg()));
+ for (unsigned Op = 0; Op < KillerIns->getNumExplicitOperands(); Op++)
+ if (KillerIns->getOperand(Op).isReg() &&
+ KillerIns->getOperand(Op).isUse() &&
+ TRI->regsOverlap(KillerIns->getOperand(Op).getReg(), OldReg))
+ KillerIns->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ KillerIns->getOperand(Op).getReg()));
}
- tie(ins_defs, ins_uses) = get_uses_and_defs_for(vec_load_ins);
- war_conflicts &= ~getVGPR32Lanes(old_reg);
+ // If we renamed, we're done. If we didn't rename, we can't get
+ // around this conflict, so we're also done.
+ break;
+ }
+
+ tie(InsDefs, InsUses) = getUsesAndDefsFor(VecLoadIns);
+ WarConflicts &= ~getVGPR32Lanes(OldReg);
}
// Coda
- used_load_dest_physregs |= ins_defs;
- used_load_source_physregs |= ins_uses;
- all_vector_loads.pop_back();
+ UsedLoadDestPhysregs |= InsDefs;
+ UsedLoadSourcePhysregs |= InsUses;
+ AllVectorLoads.pop_back();
}
// Scavenge free VGPRs and rename along each address def chain so the chains
// become register-disjoint, staying within the budget. The downstream
// post-RA load-cluster scheduler then reorders the now independent loads into
// a burst.
-
- return to_return;
+
+ return ToReturn;
}
bool GCNBreakLoadClusterDepsImpl::run(MachineFunction &MF) {
@@ -296,17 +305,17 @@ bool GCNBreakLoadClusterDepsImpl::run(MachineFunction &MF) {
MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled()
? MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize()
: false;
- occupancy_budget = ST->getMaxNumVGPRs(
+ OccupancyBudget = ST->getMaxNumVGPRs(
ST->getOccupancyWithNumVGPRs(
TRI->getNumUsedPhysRegs(*MRI, AMDGPU::VGPR_32RegClass),
DynamicBlockSize),
DynamicBlockSize);
- bool to_return = false;
+ bool ToReturn = false;
for (MachineBasicBlock &MBB : MF)
- to_return |= runOnMachineBasicBlock(MBB);
-
- return to_return;
+ ToReturn |= runOnMachineBasicBlock(MBB);
+
+ return ToReturn;
}
bool GCNBreakLoadClusterDepsLegacy::runOnMachineFunction(MachineFunction &MF) {
diff --git a/llvm/test/CodeGen/AMDGPU/break-load-cluster-deps.ll b/llvm/test/CodeGen/AMDGPU/break-load-cluster-deps.ll
new file mode 100644
index 0000000000000..b963dc687d892
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/break-load-cluster-deps.ll
@@ -0,0 +1,276 @@
+; RUN: llc -mcpu=gfx90a < %s | FileCheck %s
+; CHECK-LABEL: This Inner Loop
+; CHECK: global_load_dwordx4
+; CHECK-NEXT: global_load_dwordx4
+; CHECK-NEXT: add
+; CHECK-NEXT: global_load_dwordx4
+; CHECK-NEXT: add
+; CHECK-NEXT: global_load_dwordx4
+; CHECK-LABEL: This Inner Loop
+; CHECK: global_load_dwordx4
+; CHECK-NEXT: global_load_dwordx4
+; CHECK-NEXT: add
+; CHECK-NEXT: global_load_dwordx4
+; CHECK-NEXT: add
+; CHECK-NEXT: global_load_dwordx4
+; ModuleID = 'repro_hipperf_uavreadspeed.cpp'
+source_filename = "repro_hipperf_uavreadspeed.cpp"
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+%struct.HIP_vector_type = type { %struct.HIP_vector_base }
+%struct.HIP_vector_base = type { float, float, float, float }
+%struct.HIP_vector_type.0 = type { %struct.HIP_vector_base.1 }
+%struct.HIP_vector_base.1 = type { double, double }
+
+$_Z12uavReadSpeedI15HIP_vector_typeIfLj4EEEvPT_S3_PKjjS2_ = comdat any
+
+$_Z12uavReadSpeedI15HIP_vector_typeIdLj2EEEvPT_S3_PKjjS2_ = comdat any
+
+ at __hip_cuid_6dcd12a76005a3fa = addrspace(1) global i8 0
+ at llvm.compiler.used = appending addrspace(1) global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @__hip_cuid_6dcd12a76005a3fa to ptr)], section "llvm.metadata"
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define protected amdgpu_kernel void @_Z12uavReadSpeedI15HIP_vector_typeIfLj4EEEvPT_S3_PKjjS2_(ptr addrspace(1) noalias nofree noundef readonly captures(none) %0, ptr addrspace(1) noalias nofree noundef writeonly captures(none) %1, ptr addrspace(1) noalias nofree noundef readonly captures(none) %2, i32 noundef %3, ptr addrspace(4) nofree noundef readnone byref(%struct.HIP_vector_type) align 16 captures(none) %4) local_unnamed_addr #0 comdat {
+ %6 = tail call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+ %7 = getelementptr inbounds nuw i8, ptr addrspace(4) %6, i64 12
+ %8 = load i16, ptr addrspace(4) %7, align 4, !range !16, !invariant.load !17, !noundef !17
+ %9 = zext nneg i16 %8 to i32
+ %10 = tail call noundef i32 @llvm.amdgcn.workgroup.id.x()
+ %11 = mul i32 %10, %9
+ %12 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %13 = add i32 %11, %12
+ %14 = icmp eq i32 %3, 1
+ br i1 %14, label %15, label %22
+
+15: ; preds = %5
+ %16 = load i32, ptr addrspace(1) %2, align 4, !tbaa !18
+ %17 = urem i32 %13, %16
+ %18 = zext i32 %17 to i64
+ %19 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %18
+ %20 = sext i32 %13 to i64
+ %21 = getelementptr inbounds [16 x i8], ptr addrspace(1) %1, i64 %20
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 16 dereferenceable(16) %21, ptr addrspace(1) noundef align 16 dereferenceable(16) %19, i64 16, i1 false)
+ br label %83
+
+22: ; preds = %5
+ %23 = lshr i32 %3, 2
+ %24 = icmp eq i32 %23, 0
+ br i1 %24, label %44, label %25
+
+25: ; preds = %22
+ %26 = load i32, ptr addrspace(1) %2, align 4, !tbaa !18
+ %27 = urem i32 %13, %26
+ %28 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 12
+ %29 = load <2 x i32>, ptr addrspace(1) %28, align 4, !tbaa !18
+ %30 = insertelement <2 x i32> poison, i32 %27, i64 0
+ %31 = shufflevector <2 x i32> %30, <2 x i32> poison, <2 x i32> zeroinitializer
+ %32 = add <2 x i32> %29, %31
+ %33 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 4
+ %34 = load <2 x i32>, ptr addrspace(1) %33, align 4, !tbaa !18
+ %35 = add <2 x i32> %34, %31
+ %36 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 20
+ %37 = load i32, ptr addrspace(1) %36, align 4, !tbaa !18
+ %38 = insertelement <2 x i32> poison, i32 %37, i64 0
+ %39 = shufflevector <2 x i32> %38, <2 x i32> poison, <2 x i32> zeroinitializer
+ br label %51
+
+40: ; preds = %51
+ %41 = fadd contract <4 x float> %63, %68
+ %42 = fadd contract <4 x float> %73, %41
+ %43 = fadd contract <4 x float> %78, %42
+ br label %44
+
+44: ; preds = %40, %22
+ %45 = phi <4 x float> [ zeroinitializer, %22 ], [ %43, %40 ]
+ %46 = sext i32 %13 to i64
+ %47 = getelementptr inbounds [16 x i8], ptr addrspace(1) %1, i64 %46
+ %48 = shufflevector <4 x float> %45, <4 x float> poison, <2 x i32> <i32 0, i32 1>
+ store <2 x float> %48, ptr addrspace(1) %47, align 16
+ %49 = getelementptr inbounds nuw i8, ptr addrspace(1) %47, i64 8
+ %50 = shufflevector <4 x float> %45, <4 x float> poison, <2 x i32> <i32 2, i32 3>
+ store <2 x float> %50, ptr addrspace(1) %49, align 8
+ br label %83
+
+51: ; preds = %25, %51
+ %52 = phi i32 [ 0, %25 ], [ %81, %51 ]
+ %53 = phi <4 x float> [ zeroinitializer, %25 ], [ %78, %51 ]
+ %54 = phi <4 x float> [ zeroinitializer, %25 ], [ %73, %51 ]
+ %55 = phi <4 x float> [ zeroinitializer, %25 ], [ %68, %51 ]
+ %56 = phi <4 x float> [ zeroinitializer, %25 ], [ %63, %51 ]
+ %57 = phi <2 x i32> [ %35, %25 ], [ %79, %51 ]
+ %58 = phi <2 x i32> [ %32, %25 ], [ %80, %51 ]
+ %59 = extractelement <2 x i32> %57, i64 0
+ %60 = zext i32 %59 to i64
+ %61 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %60
+ %62 = load <4 x float>, ptr addrspace(1) %61, align 16, !tbaa !19
+ %63 = fadd contract <4 x float> %56, %62
+ %64 = extractelement <2 x i32> %57, i64 1
+ %65 = zext i32 %64 to i64
+ %66 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %65
+ %67 = load <4 x float>, ptr addrspace(1) %66, align 16, !tbaa !19
+ %68 = fadd contract <4 x float> %55, %67
+ %69 = extractelement <2 x i32> %58, i64 0
+ %70 = zext i32 %69 to i64
+ %71 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %70
+ %72 = load <4 x float>, ptr addrspace(1) %71, align 16, !tbaa !19
+ %73 = fadd contract <4 x float> %54, %72
+ %74 = extractelement <2 x i32> %58, i64 1
+ %75 = zext i32 %74 to i64
+ %76 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %75
+ %77 = load <4 x float>, ptr addrspace(1) %76, align 16, !tbaa !19
+ %78 = fadd contract <4 x float> %53, %77
+ %79 = add <2 x i32> %39, %57
+ %80 = add <2 x i32> %39, %58
+ %81 = add nuw nsw i32 %52, 1
+ %82 = icmp eq i32 %81, %23
+ br i1 %82, label %40, label %51, !llvm.loop !20
+
+83: ; preds = %44, %15
+ ret void
+}
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define protected amdgpu_kernel void @_Z12uavReadSpeedI15HIP_vector_typeIdLj2EEEvPT_S3_PKjjS2_(ptr addrspace(1) noalias nofree noundef readonly captures(none) %0, ptr addrspace(1) noalias nofree noundef writeonly captures(none) %1, ptr addrspace(1) noalias nofree noundef readonly captures(none) %2, i32 noundef %3, ptr addrspace(4) nofree noundef readnone byref(%struct.HIP_vector_type.0) align 16 captures(none) %4) local_unnamed_addr #0 comdat {
+ %6 = tail call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+ %7 = getelementptr inbounds nuw i8, ptr addrspace(4) %6, i64 12
+ %8 = load i16, ptr addrspace(4) %7, align 4, !range !16, !invariant.load !17, !noundef !17
+ %9 = zext nneg i16 %8 to i32
+ %10 = tail call noundef i32 @llvm.amdgcn.workgroup.id.x()
+ %11 = mul i32 %10, %9
+ %12 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %13 = add i32 %11, %12
+ %14 = icmp eq i32 %3, 1
+ br i1 %14, label %15, label %22
+
+15: ; preds = %5
+ %16 = load i32, ptr addrspace(1) %2, align 4, !tbaa !18
+ %17 = urem i32 %13, %16
+ %18 = zext i32 %17 to i64
+ %19 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %18
+ %20 = sext i32 %13 to i64
+ %21 = getelementptr inbounds [16 x i8], ptr addrspace(1) %1, i64 %20
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 16 dereferenceable(16) %21, ptr addrspace(1) noundef align 16 dereferenceable(16) %19, i64 16, i1 false)
+ br label %83
+
+22: ; preds = %5
+ %23 = lshr i32 %3, 2
+ %24 = icmp eq i32 %23, 0
+ br i1 %24, label %44, label %25
+
+25: ; preds = %22
+ %26 = load i32, ptr addrspace(1) %2, align 4, !tbaa !18
+ %27 = urem i32 %13, %26
+ %28 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 12
+ %29 = load <2 x i32>, ptr addrspace(1) %28, align 4, !tbaa !18
+ %30 = insertelement <2 x i32> poison, i32 %27, i64 0
+ %31 = shufflevector <2 x i32> %30, <2 x i32> poison, <2 x i32> zeroinitializer
+ %32 = add <2 x i32> %29, %31
+ %33 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 4
+ %34 = load <2 x i32>, ptr addrspace(1) %33, align 4, !tbaa !18
+ %35 = add <2 x i32> %34, %31
+ %36 = getelementptr inbounds nuw i8, ptr addrspace(1) %2, i64 20
+ %37 = load i32, ptr addrspace(1) %36, align 4, !tbaa !18
+ %38 = insertelement <2 x i32> poison, i32 %37, i64 0
+ %39 = shufflevector <2 x i32> %38, <2 x i32> poison, <2 x i32> zeroinitializer
+ br label %51
+
+40: ; preds = %51
+ %41 = fadd contract <2 x double> %63, %68
+ %42 = fadd contract <2 x double> %73, %41
+ %43 = fadd contract <2 x double> %78, %42
+ br label %44
+
+44: ; preds = %40, %22
+ %45 = phi <2 x double> [ zeroinitializer, %22 ], [ %43, %40 ]
+ %46 = extractelement <2 x double> %45, i64 0
+ %47 = extractelement <2 x double> %45, i64 1
+ %48 = sext i32 %13 to i64
+ %49 = getelementptr inbounds [16 x i8], ptr addrspace(1) %1, i64 %48
+ store double %46, ptr addrspace(1) %49, align 16
+ %50 = getelementptr inbounds nuw i8, ptr addrspace(1) %49, i64 8
+ store double %47, ptr addrspace(1) %50, align 8
+ br label %83
+
+51: ; preds = %25, %51
+ %52 = phi i32 [ 0, %25 ], [ %81, %51 ]
+ %53 = phi <2 x double> [ zeroinitializer, %25 ], [ %78, %51 ]
+ %54 = phi <2 x double> [ zeroinitializer, %25 ], [ %73, %51 ]
+ %55 = phi <2 x double> [ zeroinitializer, %25 ], [ %68, %51 ]
+ %56 = phi <2 x double> [ zeroinitializer, %25 ], [ %63, %51 ]
+ %57 = phi <2 x i32> [ %35, %25 ], [ %79, %51 ]
+ %58 = phi <2 x i32> [ %32, %25 ], [ %80, %51 ]
+ %59 = extractelement <2 x i32> %57, i64 0
+ %60 = zext i32 %59 to i64
+ %61 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %60
+ %62 = load <2 x double>, ptr addrspace(1) %61, align 16, !tbaa !19
+ %63 = fadd contract <2 x double> %56, %62
+ %64 = extractelement <2 x i32> %57, i64 1
+ %65 = zext i32 %64 to i64
+ %66 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %65
+ %67 = load <2 x double>, ptr addrspace(1) %66, align 16, !tbaa !19
+ %68 = fadd contract <2 x double> %55, %67
+ %69 = extractelement <2 x i32> %58, i64 0
+ %70 = zext i32 %69 to i64
+ %71 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %70
+ %72 = load <2 x double>, ptr addrspace(1) %71, align 16, !tbaa !19
+ %73 = fadd contract <2 x double> %54, %72
+ %74 = extractelement <2 x i32> %58, i64 1
+ %75 = zext i32 %74 to i64
+ %76 = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %0, i64 %75
+ %77 = load <2 x double>, ptr addrspace(1) %76, align 16, !tbaa !19
+ %78 = fadd contract <2 x double> %53, %77
+ %79 = add <2 x i32> %39, %57
+ %80 = add <2 x i32> %39, %58
+ %81 = add nuw nsw i32 %52, 1
+ %82 = icmp eq i32 %81, %23
+ br i1 %82, label %40, label %51, !llvm.loop !22
+
+83: ; preds = %44, %15
+ ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg) #2
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,1024" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-no-wwm" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx90a" "uniform-work-group-size" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+!llvm.errno.tbaa = !{!5, !10}
+!opencl.ocl.version = !{!15}
+
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
+!1 = !{i32 1, !"amdgpu_printf_kind", !"hostcall"}
+!2 = !{i32 8, !"PIC Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"AMD clang version 24.0.0git (https://github.com/ROCm/llvm-project.git 657cfa16903ad4c5921a6f8992bb50009da0256f)"}
+!5 = !{!6, !7, i64 0}
+!6 = !{!"__libc_errno", !7, i64 0}
+!7 = !{!"int", !8, i64 0}
+!8 = !{!"omnipotent char", !9, i64 0}
+!9 = !{!"Simple C++ TBAA"}
+!10 = !{!11, !12, i64 0}
+!11 = !{!"__libc_errno", !12, i64 0}
+!12 = !{!"int", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !{i32 2, i32 0}
+!16 = !{i16 1, i16 1025}
+!17 = !{}
+!18 = !{!7, !7, i64 0}
+!19 = !{!8, !8, i64 0}
+!20 = distinct !{!20, !21}
+!21 = !{!"llvm.loop.mustprogress"}
+!22 = distinct !{!22, !21}
>From 181a2809a3008e4b7c3400b0791551c035bd7055 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 14 Aug 2026 15:00:08 -0500
Subject: [PATCH 07/10] Don't use a register if it just creates another WAR for
the cluster
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 0f8f15e38fea5..fd9e65419fef4 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -236,6 +236,18 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
AccumIt != MBB.rend(); ++AccumIt)
LRU.accumulate(*AccumIt);
+ // If it's used by a load that could be in our cluster, it's _NOT_ free.
+ bitset<AMDGPU::NUM_TARGET_REGS> BannedRegs =
+ UsedLoadDestPhysregs | InsDefs;
+ for (auto VecIt = AllVectorLoads.rbegin();
+ VecIt != AllVectorLoads.rend(); VecIt++) {
+ bitset<AMDGPU::NUM_TARGET_REGS> FutureInsDefs =
+ getUsesAndDefsFor(**VecIt).first;
+ if ((FutureInsDefs & BannedRegs).any())
+ break;
+ BannedRegs |= FutureInsDefs;
+ }
+
// Iterate over registers in physical register class
const TargetRegisterClass &DefinedRegClass =
*TRI->getPhysRegBaseClass(OldReg);
@@ -243,7 +255,9 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
for (I = 0; I < DefinedRegClass.getRegisters().size() &&
I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget;
I++)
- if (LRU.available(DefinedRegClass.getRegisters()[I]))
+ if (LRU.available(DefinedRegClass.getRegisters()[I]) &&
+ (getVGPR32Lanes(DefinedRegClass.getRegisters()[I]) & BannedRegs)
+ .none())
break;
// Actually rename the register
>From 371079fc6547eb77ce2b0b8af110f3e3b7c68b58 Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Fri, 14 Aug 2026 15:11:04 -0500
Subject: [PATCH 08/10] off-by-one
---
llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index fd9e65419fef4..85a639ea37915 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -239,7 +239,7 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
// If it's used by a load that could be in our cluster, it's _NOT_ free.
bitset<AMDGPU::NUM_TARGET_REGS> BannedRegs =
UsedLoadDestPhysregs | InsDefs;
- for (auto VecIt = AllVectorLoads.rbegin();
+ for (auto VecIt = std::next(AllVectorLoads.rbegin());
VecIt != AllVectorLoads.rend(); VecIt++) {
bitset<AMDGPU::NUM_TARGET_REGS> FutureInsDefs =
getUsesAndDefsFor(**VecIt).first;
>From 30131cc8fd5b0051ca6ca8b5202fb0fae360b10f Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 17 Aug 2026 16:54:09 -0500
Subject: [PATCH 09/10] Intermediate commit before enabling new logic
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 210 ++++++++++++++++--
1 file changed, 193 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 85a639ea37915..25ddcc7ebcb24 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -69,7 +69,16 @@ class GCNBreakLoadClusterDepsImpl {
bitset<AMDGPU::NUM_TARGET_REGS> getVGPR32Lanes(Register Reg) const;
pair<bitset<AMDGPU::NUM_TARGET_REGS>, bitset<AMDGPU::NUM_TARGET_REGS>>
getUsesAndDefsFor(MachineInstr &MI) const;
+ Register promoteToSuperRegister(MachineInstr &MI, Register SubReg, bool Defs,
+ bool Uses);
Register renameRegister(Register FromReg, Register ToReg, Register RenameReg);
+ bool
+ findReplaceRegisterOperand(MachineInstr &MI, unsigned OpNum,
+ const bitset<AMDGPU::NUM_TARGET_REGS> &BannedRegs);
+ bool isVGPRLoad(MachineInstr &MI) const {
+ return MI.mayLoad() && MI.getOperand(0).isReg() &&
+ TRI->isVGPR(*MRI, MI.getOperand(0).getReg());
+ }
public:
bool run(MachineFunction &MF);
@@ -126,6 +135,19 @@ GCNBreakLoadClusterDepsImpl::getUsesAndDefsFor(MachineInstr &MI) const {
return ToReturn;
}
+Register GCNBreakLoadClusterDepsImpl::promoteToSuperRegister(MachineInstr &MI,
+ Register SubReg,
+ bool Defs,
+ bool Uses) {
+ for (MachineOperand &Operand : MI.explicit_operands())
+ if (Operand.isReg() && (Defs || Operand.isUse()) &&
+ (Uses || Operand.isDef()) &&
+ TRI->isSuperRegister(SubReg, Operand.getReg()))
+ SubReg = Operand.getReg();
+
+ return SubReg;
+}
+
Register GCNBreakLoadClusterDepsImpl::renameRegister(Register FromReg,
Register ToReg,
Register RenameReg) {
@@ -137,6 +159,139 @@ Register GCNBreakLoadClusterDepsImpl::renameRegister(Register FromReg,
return RenameReg;
}
+bool GCNBreakLoadClusterDepsImpl::findReplaceRegisterOperand(
+ MachineInstr &MI, unsigned OpNum,
+ const bitset<AMDGPU::NUM_TARGET_REGS> &BannedRegs) {
+ MachineBasicBlock& MBB = *MI.getParent();
+ MachineInstr *DefToRename = nullptr, *KillerIns = nullptr;
+ Register OldReg = MI.getOperand(OpNum).getReg();
+ bitset<AMDGPU::NUM_TARGET_REGS> OldRegClobbers = getVGPR32Lanes(OldReg);
+ if (MI.getOperand(OpNum).isDef())
+ DefToRename = &MI;
+ else
+ KillerIns = &MI;
+
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
+
+ // First, go forward from def to find the kill
+ if (DefToRename) {
+ bitset<AMDGPU::NUM_TARGET_REGS> ClobberedSubregs;
+ MachineInstr *NewKiller = KillerIns ? KillerIns : nullptr;
+ Register OldOldReg = OldReg;
+ for (MachineBasicBlock::iterator It =
+ std::next(DefToRename->getIterator());
+ (OldRegClobbers & ~ClobberedSubregs).any() && It != MBB.end();
+ ++It) {
+ auto Subregs = getUsesAndDefsFor(*It);
+ if ((Subregs.second & OldRegClobbers).any())
+ NewKiller = &*It;
+ ClobberedSubregs |= Subregs.first;
+
+ //Handle promoting OldReg to a super-register of it
+ Register NewOldReg = promoteToSuperRegister(*std::prev(It),OldReg,true,false);
+ NewOldReg = promoteToSuperRegister(*It, NewOldReg, false, true);
+ if (NewOldReg != OldReg) {
+ Changed = true;
+ OldReg = NewOldReg;
+ OldRegClobbers = getVGPR32Lanes(OldReg);
+ }
+ }
+
+ if (NewKiller != KillerIns) {
+ KillerIns = NewKiller;
+ Changed = true;
+ }
+
+ // Are we live out with no true kill? Fail: we can't do this with a
+ // block-local analysis.
+ if (OldOldReg == OldReg && (OldRegClobbers & ~ClobberedSubregs).any()) {
+ LiveRegUnits LRU(*TRI);
+ LRU.addLiveOuts(MBB);
+ if (!LRU.available(OldReg))
+ return false;
+ }
+ }
+
+ //Second, go backward from killer to find the def
+ if (KillerIns) {
+ bitset<AMDGPU::NUM_TARGET_REGS> ClobberedSubregs;
+ MachineInstr *NewDef = DefToRename ? DefToRename : nullptr;
+ for (MachineBasicBlock::reverse_iterator RIt =
+ std::next(KillerIns->getReverseIterator());
+ RIt != MBB.rend(); ++RIt) {
+ if (RIt->modifiesRegister(OldReg, TRI))
+ ClobberedSubregs |= getUsesAndDefsFor(*RIt).first;
+
+ if ((OldRegClobbers & ~ClobberedSubregs).none()) {
+ NewDef = &*RIt;
+ break;
+ }
+ }
+
+ if (NewDef != DefToRename) {
+ DefToRename = NewDef;
+ Changed = true;
+ }
+ }
+ }
+
+ // Now, perform the rename between (DefToRename, KillerIns)
+
+ // Find a free reg
+ LiveRegUnits LRU(*TRI);
+ LRU.addLiveOuts(MBB);
+ for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin();
+ &*LiveRIt != &*KillerIns; ++LiveRIt)
+ LRU.stepBackward(*LiveRIt);
+ for (MachineBasicBlock::reverse_iterator AccumIt =
+ KillerIns->getReverseIterator();
+ AccumIt != MBB.rend(); ++AccumIt)
+ LRU.accumulate(*AccumIt);
+
+ // Iterate over registers in physical register class
+ const TargetRegisterClass &DefinedRegClass =
+ *TRI->getPhysRegBaseClass(OldReg);
+ unsigned I;
+ for (I = 0; I < DefinedRegClass.getRegisters().size() &&
+ I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget;
+ I++)
+ if (LRU.available(DefinedRegClass.getRegisters()[I]) &&
+ (getVGPR32Lanes(DefinedRegClass.getRegisters()[I]) & BannedRegs).none())
+ break;
+
+ // Fail if we couldn't find a suitable free register
+ if (I == DefinedRegClass.getRegisters().size() ||
+ I * DefinedRegClass.getSizeInBits() / 32 >= OccupancyBudget)
+ return false;
+
+ // Actually rename the register
+ for (unsigned Op = 0; Op < DefToRename->getNumExplicitOperands(); Op++)
+ if (DefToRename->getOperand(Op).isReg() && DefToRename->getOperand(Op).isDef() &&
+ TRI->regsOverlap(DefToRename->getOperand(Op).getReg(), OldReg))
+ DefToRename->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ DefToRename->getOperand(Op).getReg()));
+ for (MachineBasicBlock::iterator RenameIt =
+ std::next(DefToRename->getIterator());
+ RenameIt != KillerIns; ++RenameIt)
+ for (unsigned Op = 0; Op < RenameIt->getNumExplicitOperands(); Op++)
+ if (RenameIt->getOperand(Op).isReg() &&
+ TRI->regsOverlap(RenameIt->getOperand(Op).getReg(), OldReg))
+ RenameIt->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ RenameIt->getOperand(Op).getReg()));
+ for (unsigned Op = 0; Op < KillerIns->getNumExplicitOperands(); Op++)
+ if (KillerIns->getOperand(Op).isReg() &&
+ KillerIns->getOperand(Op).isUse() &&
+ TRI->regsOverlap(KillerIns->getOperand(Op).getReg(), OldReg))
+ KillerIns->getOperand(Op).setReg(
+ renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
+ KillerIns->getOperand(Op).getReg()));
+ return true;
+}
+
bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
MachineBasicBlock &MBB) {
bool ToReturn = false;
@@ -146,25 +301,58 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
// common scratch register (WAR/WAW anti-dependencies).
vector<MachineInstr *> AllVectorLoads;
for (MachineInstr &MI : MBB)
- if (MI.mayLoad() && MI.getOperand(0).isReg() &&
- TRI->isVGPR(*MRI, MI.getOperand(0).getReg()))
+ if (isVGPRLoad(MI))
AllVectorLoads.push_back(&MI);
reverse(AllVectorLoads.begin(), AllVectorLoads.end()); // efficiency
bitset<AMDGPU::NUM_TARGET_REGS> UsedLoadSourcePhysregs, UsedLoadDestPhysregs;
+ unordered_set<MachineInstr*> ClusterLoads;
while (!AllVectorLoads.empty()) {
MachineInstr &VecLoadIns = *AllVectorLoads.back();
bitset<AMDGPU::NUM_TARGET_REGS> InsDefs, InsUses;
tie(InsDefs, InsUses) = getUsesAndDefsFor(VecLoadIns);
- // This means the load is not independent from previous loads
- if ((InsDefs & UsedLoadDestPhysregs).any()) {
+ if (ClusterLoads.size() && !ClusterLoads.count(&VecLoadIns)) {
+ ClusterLoads.clear();
UsedLoadSourcePhysregs.reset();
UsedLoadDestPhysregs.reset();
AllVectorLoads.pop_back();
continue;
- }
+ } else if (!ClusterLoads.count(&VecLoadIns)) {
+ bitset<AMDGPU::NUM_TARGET_REGS> ClusterRAWHazards;
+ for (MachineBasicBlock::iterator ForwardIt = VecLoadIns.getIterator();
+ ForwardIt != MBB.end(); ++ForwardIt) {
+ if (isVGPRLoad(*ForwardIt)) {
+ bitset<AMDGPU::NUM_TARGET_REGS> UsedVGPRs;
+ for (MachineOperand &Operand : ForwardIt->uses())
+ if (Operand.isReg() && Operand.isUse() &&
+ TRI->isVGPR(*MRI, Operand.getReg()))
+ UsedVGPRs |= getVGPR32Lanes(Operand.getReg());
+
+ if ((ClusterRAWHazards & UsedVGPRs).any())
+ break;
+ ClusterLoads.insert(&*ForwardIt);
+ ClusterRAWHazards |=
+ getVGPR32Lanes(ForwardIt->getOperand(0).getReg());
+ } else
+ for (MachineOperand &Operand : ForwardIt->defs())
+ if (TRI->isVGPR(*MRI,Operand.getReg()))
+ ClusterRAWHazards &= getVGPR32Lanes(Operand.getReg());
+ }
+ } else
+ ClusterLoads.erase(&VecLoadIns);
+
+ // If it's used or defined by a load that could be in our cluster, it's
+ // _NOT_ free.
+ bitset<AMDGPU::NUM_TARGET_REGS> BannedRegs =
+ UsedLoadDestPhysregs | UsedLoadSourcePhysregs;
+ for (MachineInstr *FutureVecLoad : ClusterLoads) {
+ auto UsesAndDefs = getUsesAndDefsFor(*FutureVecLoad);
+ BannedRegs |= UsesAndDefs.first;
+ BannedRegs |= UsesAndDefs.second;
+ }
+
// Check if we have something to rename
bitset<AMDGPU::NUM_TARGET_REGS> WarConflicts =
InsUses & UsedLoadSourcePhysregs;
@@ -236,18 +424,6 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
AccumIt != MBB.rend(); ++AccumIt)
LRU.accumulate(*AccumIt);
- // If it's used by a load that could be in our cluster, it's _NOT_ free.
- bitset<AMDGPU::NUM_TARGET_REGS> BannedRegs =
- UsedLoadDestPhysregs | InsDefs;
- for (auto VecIt = std::next(AllVectorLoads.rbegin());
- VecIt != AllVectorLoads.rend(); VecIt++) {
- bitset<AMDGPU::NUM_TARGET_REGS> FutureInsDefs =
- getUsesAndDefsFor(**VecIt).first;
- if ((FutureInsDefs & BannedRegs).any())
- break;
- BannedRegs |= FutureInsDefs;
- }
-
// Iterate over registers in physical register class
const TargetRegisterClass &DefinedRegClass =
*TRI->getPhysRegBaseClass(OldReg);
>From 9eb325af0496ad4ad79aaa678bf59e225a51954f Mon Sep 17 00:00:00 2001
From: Patrick Simmons <patrick.simmons at amd.com>
Date: Mon, 17 Aug 2026 20:21:58 -0500
Subject: [PATCH 10/10] Finish more aggressive implementation
---
.../Target/AMDGPU/GCNBreakLoadClusterDeps.cpp | 156 +++++-------------
1 file changed, 39 insertions(+), 117 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
index 25ddcc7ebcb24..ff2f7fd63675e 100644
--- a/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNBreakLoadClusterDeps.cpp
@@ -247,23 +247,24 @@ bool GCNBreakLoadClusterDepsImpl::findReplaceRegisterOperand(
LRU.stepBackward(*LiveRIt);
for (MachineBasicBlock::reverse_iterator AccumIt =
KillerIns->getReverseIterator();
- AccumIt != MBB.rend(); ++AccumIt)
+ &*AccumIt != DefToRename; ++AccumIt)
LRU.accumulate(*AccumIt);
// Iterate over registers in physical register class
const TargetRegisterClass &DefinedRegClass =
*TRI->getPhysRegBaseClass(OldReg);
unsigned I;
- for (I = 0; I < DefinedRegClass.getRegisters().size() &&
- I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget;
- I++)
+ for (I = 0; I < DefinedRegClass.getRegisters().size(); I++) {
+ if (TRI->getHWRegIndex(DefinedRegClass.getRegisters()[I]) >=
+ OccupancyBudget)
+ continue;
if (LRU.available(DefinedRegClass.getRegisters()[I]) &&
(getVGPR32Lanes(DefinedRegClass.getRegisters()[I]) & BannedRegs).none())
break;
+ }
// Fail if we couldn't find a suitable free register
- if (I == DefinedRegClass.getRegisters().size() ||
- I * DefinedRegClass.getSizeInBits() / 32 >= OccupancyBudget)
+ if (I == DefinedRegClass.getRegisters().size())
return false;
// Actually rename the register
@@ -353,125 +354,46 @@ bool GCNBreakLoadClusterDepsImpl::runOnMachineBasicBlock(
BannedRegs |= UsesAndDefs.second;
}
- // Check if we have something to rename
+ // Check if we have something to rename due to WAR
bitset<AMDGPU::NUM_TARGET_REGS> WarConflicts =
- InsUses & UsedLoadSourcePhysregs;
+ (InsUses | InsDefs) & (UsedLoadSourcePhysregs | UsedLoadDestPhysregs);
while (WarConflicts.any()) {
Register OldReg = WarConflicts._Find_first();
- bitset<AMDGPU::NUM_TARGET_REGS> OldRegWarConflicts;
- for (const MachineOperand &Operand : VecLoadIns.operands())
- if (Operand.isReg() && Operand.isUse() &&
- TRI->regsOverlap(OldReg, Operand.getReg())) {
- OldRegWarConflicts |= getVGPR32Lanes(Operand.getReg());
- if (TRI->getPhysRegBaseClass(Operand.getReg())->getSizeInBits() >
- TRI->getPhysRegBaseClass(OldReg)->getSizeInBits())
- OldReg = Operand.getReg();
- }
-
- bitset<AMDGPU::NUM_TARGET_REGS> RitRegWarConflicts;
- for (MachineBasicBlock::reverse_iterator RIt =
- ++VecLoadIns.getReverseIterator();
- RIt != MBB.rend(); ++RIt) {
- if (RIt->modifiesRegister(OldReg, TRI))
- RitRegWarConflicts |= getUsesAndDefsFor(*RIt).first;
-
- if ((OldRegWarConflicts & ~RitRegWarConflicts).any())
- continue;
-
- // First, make sure we don't modify EXEC before redefining register.
- bool ExecModified = false, Redefined = false;
- for (MachineBasicBlock::iterator It = std::next(RIt->getIterator());
- It != MBB.end(); ++It)
- if (It->definesRegister(AMDGPU::EXEC, TRI)) {
- ExecModified = true;
- break;
- } else if (It->modifiesRegister(OldReg, TRI)) {
- Redefined = true;
- }
-
- // Can't do anything if EXEC modified
- if (ExecModified)
+ unsigned OpNum;
+ for (OpNum = 0; OpNum < VecLoadIns.getNumExplicitOperands(); OpNum++)
+ if (VecLoadIns.getOperand(OpNum).isReg() &&
+ TRI->regsOverlap(OldReg, VecLoadIns.getOperand(OpNum).getReg()))
break;
-
- LiveRegUnits LRU(*TRI);
- LRU.addLiveOuts(MBB);
-
- // If we're live out of the block and the conflicing reg hasn't been
- // redefined, we can't do this with a block-local analysis.
- if (!Redefined && !LRU.available(OldReg))
+ assert(OpNum != VecLoadIns.getNumExplicitOperands() &&
+ "There should be a conflicting register operand. Where is it?");
+ if (!findReplaceRegisterOperand(VecLoadIns, OpNum, BannedRegs))
+ break;
+
+ tie(InsDefs, InsUses) = getUsesAndDefsFor(VecLoadIns);
+ WarConflicts =
+ (InsUses | InsDefs) & (UsedLoadSourcePhysregs | UsedLoadDestPhysregs);
+ }
+
+ // Check if we have something to rename due to WAR
+ bitset<AMDGPU::NUM_TARGET_REGS> SelfConflicts = InsUses & InsDefs;
+ while (SelfConflicts.any()) {
+ Register OldReg = SelfConflicts._Find_first();
+ unsigned OpNum;
+ for (OpNum = 0; OpNum < VecLoadIns.getNumExplicitOperands(); OpNum++)
+ if (VecLoadIns.getOperand(OpNum).isReg() &&
+ VecLoadIns.getOperand(OpNum).isUse() &&
+ TRI->regsOverlap(OldReg, VecLoadIns.getOperand(OpNum).getReg()))
break;
-
- // Find the instruction which kills the def in RIt
- bitset<AMDGPU::NUM_TARGET_REGS> KilledSubregs;
- MachineBasicBlock::iterator KillerIns = VecLoadIns;
- for (MachineBasicBlock::iterator CandidateKiller = KillerIns;
- CandidateKiller != MBB.end(); ++CandidateKiller) {
- if (CandidateKiller->modifiesRegister(OldReg, TRI)) {
- KilledSubregs |= getUsesAndDefsFor(*CandidateKiller).first;
- if ((RitRegWarConflicts & ~KilledSubregs).none())
- break;
- }
- if (CandidateKiller->readsRegister(OldReg, TRI))
- KillerIns = CandidateKiller;
- }
-
- // See what's free
- for (MachineBasicBlock::reverse_iterator LiveRIt = MBB.rbegin();
- &*LiveRIt != &*KillerIns; ++LiveRIt)
- LRU.stepBackward(*LiveRIt);
- for (MachineBasicBlock::reverse_iterator AccumIt =
- KillerIns->getReverseIterator();
- AccumIt != MBB.rend(); ++AccumIt)
- LRU.accumulate(*AccumIt);
-
- // Iterate over registers in physical register class
- const TargetRegisterClass &DefinedRegClass =
- *TRI->getPhysRegBaseClass(OldReg);
- unsigned I;
- for (I = 0; I < DefinedRegClass.getRegisters().size() &&
- I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget;
- I++)
- if (LRU.available(DefinedRegClass.getRegisters()[I]) &&
- (getVGPR32Lanes(DefinedRegClass.getRegisters()[I]) & BannedRegs)
- .none())
- break;
-
- // Actually rename the register
- if (I < DefinedRegClass.getRegisters().size() &&
- I * DefinedRegClass.getSizeInBits() / 32 < OccupancyBudget) {
- for (unsigned Op = 0; Op < RIt->getNumExplicitOperands(); Op++)
- if (RIt->getOperand(Op).isReg() && RIt->getOperand(Op).isDef() &&
- TRI->regsOverlap(RIt->getOperand(Op).getReg(), OldReg))
- RIt->getOperand(Op).setReg(
- renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
- RIt->getOperand(Op).getReg()));
- for (MachineBasicBlock::iterator RenameIt =
- std::next(RIt->getIterator());
- RenameIt != KillerIns; ++RenameIt)
- for (unsigned Op = 0; Op < RenameIt->getNumExplicitOperands(); Op++)
- if (RenameIt->getOperand(Op).isReg() &&
- TRI->regsOverlap(RenameIt->getOperand(Op).getReg(), OldReg))
- RenameIt->getOperand(Op).setReg(
- renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
- RenameIt->getOperand(Op).getReg()));
- for (unsigned Op = 0; Op < KillerIns->getNumExplicitOperands(); Op++)
- if (KillerIns->getOperand(Op).isReg() &&
- KillerIns->getOperand(Op).isUse() &&
- TRI->regsOverlap(KillerIns->getOperand(Op).getReg(), OldReg))
- KillerIns->getOperand(Op).setReg(
- renameRegister(OldReg, DefinedRegClass.getRegisters()[I],
- KillerIns->getOperand(Op).getReg()));
- }
-
- // If we renamed, we're done. If we didn't rename, we can't get
- // around this conflict, so we're also done.
+ assert(OpNum != VecLoadIns.getNumExplicitOperands() &&
+ "There should be a conflicting register operand. Where is it?");
+ bitset<AMDGPU::NUM_TARGET_REGS> SelfBannedRegs = BannedRegs | InsDefs;
+ if (!findReplaceRegisterOperand(VecLoadIns, OpNum, SelfBannedRegs))
break;
- }
-
+
tie(InsDefs, InsUses) = getUsesAndDefsFor(VecLoadIns);
- WarConflicts &= ~getVGPR32Lanes(OldReg);
+ SelfConflicts = InsUses & InsDefs;
}
-
+
// Coda
UsedLoadDestPhysregs |= InsDefs;
UsedLoadSourcePhysregs |= InsUses;
More information about the llvm-commits
mailing list