[llvm] [AMDGPU] Add SIGlobalLoadSAddrToVAddr pass to reduce s_wait_xcnt stalls (PR #190654)
Vigneshwar Jayakumar via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 12:28:39 PDT 2026
https://github.com/VigneshwarJ updated https://github.com/llvm/llvm-project/pull/190654
>From 4eb2c4b90fb9b7214493282b5a6a8736941e18bf Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Tue, 31 Mar 2026 13:26:45 -0500
Subject: [PATCH 1/8] [AMDGPU] Add SIGlobalLoadSAddrToVAddr pass to reduce
s_wait_xcnt stalls
On GFX1250, high SGPR pressure causes the register allocator to reuse
the same SGPR pair across consecutive global loads. Since the hardware
needs the SGPR address to remain valid for address translation replay,
SIInsertWaitcnts inserts s_wait_xcnt before every SGPR redefinition,
serializing the entire load sequence.
Add a pass that converts SADDR global loads/stores to VADDR form by
copying the SGPR address into a VGPR pair when the SGPR is about to be
overwritten. VGPR defs do not require s_wait_xcnt, eliminating the
stalls. The pass runs before VGPR allocation and inserts KILL pseudos
to prevent RA from reusing the same VGPR across converted loads.
Disabled by default; enable with -amdgpu-global-load-saddr-to-vaddr=1.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 10 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 274 ++++++++++++++++++
.../Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h | 25 ++
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 200 +++++++++++++
7 files changed, 514 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
create mode 100644 llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
create mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 878f374110159..16b3d423a45ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -214,6 +214,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
+void initializeSIGlobalLoadSAddrToVAddrLegacyPass(PassRegistry &);
+extern char &SIGlobalLoadSAddrToVAddrLegacyID;
+
void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
extern char &SILoadStoreOptimizerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 8a046e83548cc..84efd6452e008 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -133,6 +133,7 @@ MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
+MACHINE_FUNCTION_PASS("si-global-load-saddr-to-vaddr", SIGlobalLoadSAddrToVAddrPass())
MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index daa9f933fce59..6a986a16a37eb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -52,6 +52,7 @@
#include "SIFixVGPRCopies.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
+#include "SIGlobalLoadSAddrToVAddr.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -727,6 +728,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
+ initializeSIGlobalLoadSAddrToVAddrLegacyPass(*PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -1814,6 +1816,8 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
+ addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1851,6 +1855,8 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
+ addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -2455,6 +2461,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
+ addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2549,6 +2557,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
+ addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index cda288e012925..1b193a4b581fb 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -161,6 +161,7 @@ add_llvm_target(AMDGPUCodeGen
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
SIFrameLowering.cpp
+ SIGlobalLoadSAddrToVAddr.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
SIInstrInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
new file mode 100644
index 0000000000000..838b22132943f
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -0,0 +1,274 @@
+//===-- SIGlobalLoadSAddrToVAddr.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Convert SADDR global loads/stores to VADDR form when the SGPR address pair
+// is about to be overwritten. This eliminates s_wait_xcnt hazards that
+// serialize loads sharing a single SGPR scratch address register.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SIGlobalLoadSAddrToVAddr.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
+
+static cl::opt<bool> EnableSAddrToVAddr(
+ "amdgpu-global-load-saddr-to-vaddr",
+ cl::desc(
+ "Convert SADDR global loads/stores to VADDR when saddr is overwritten"),
+ cl::Hidden);
+
+static cl::opt<unsigned> SAddrToVAddrWindow(
+ "amdgpu-global-load-saddr-to-vaddr-window",
+ cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
+ cl::Hidden);
+
+static bool isEnabled() {
+ if (EnableSAddrToVAddr.getNumOccurrences())
+ return EnableSAddrToVAddr;
+ if (const char *Env = std::getenv("AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR"))
+ return StringRef(Env) != "0";
+ return false;
+}
+
+namespace {
+
+class SIGlobalLoadSAddrToVAddr {
+ const SIInstrInfo *TII = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+ MachineRegisterInfo *MRI = nullptr;
+
+ bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
+ Register &NewAddrReg);
+ bool sAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+
+public:
+ bool run(MachineFunction &MF);
+};
+
+class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (!isEnabled())
+ return false;
+ SIGlobalLoadSAddrToVAddr Impl;
+ return Impl.run(MF);
+ }
+
+ StringRef getPassName() const override {
+ return "SI Global Load SAddr to VAddr";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // end anonymous namespace
+
+char SIGlobalLoadSAddrToVAddrLegacy::ID = 0;
+
+INITIALIZE_PASS(SIGlobalLoadSAddrToVAddrLegacy, DEBUG_TYPE,
+ "SI Global Load SAddr to VAddr", false, false)
+
+char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
+ SIGlobalLoadSAddrToVAddrLegacy::ID;
+
+PreservedAnalyses
+SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!isEnabled())
+ return PreservedAnalyses::all();
+ SIGlobalLoadSAddrToVAddr Impl;
+ if (!Impl.run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedInWindow(MachineInstr &MI,
+ Register SAddr) {
+ unsigned Count = 0;
+ MachineBasicBlock::iterator Next = std::next(MI.getIterator());
+ MachineBasicBlock::iterator End = MI.getParent()->end();
+ for (; Next != End && Count < SAddrToVAddrWindow; ++Next) {
+ if (Next->isMetaInstruction())
+ continue;
+ ++Count;
+ for (const MachineOperand &MO : Next->operands()) {
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ Register Reg = MO.getReg();
+ if (Reg.isPhysical() && TRI->regsOverlap(Reg, SAddr))
+ return true;
+ }
+ }
+ return false;
+}
+
+bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
+ MachineBasicBlock &MBB,
+ Register &NewAddrReg) {
+ unsigned Opc = MI.getOpcode();
+ int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
+ if (NewOpc < 0)
+ return false;
+
+ int NewVAddrIdx =
+ AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
+ if (NewVAddrIdx < 0)
+ return false;
+
+ int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
+ assert(OldSAddrIdx >= 0 && OldVAddrIdx >= 0);
+
+ MachineOperand &SAddr = MI.getOperand(OldSAddrIdx);
+ MachineOperand &VAddr = MI.getOperand(OldVAddrIdx);
+ Register SAddrReg = SAddr.getReg();
+ Register OldVAddrReg = VAddr.getReg();
+
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ bool VAddrIsZero = false;
+ if (OldVAddrReg.isVirtual()) {
+ MachineInstr *VAddrDef = MRI->getVRegDef(OldVAddrReg);
+ if (VAddrDef &&
+ (VAddrDef->isImplicitDef() ||
+ (VAddrDef->isMoveImmediate() && VAddrDef->getOperand(1).isImm() &&
+ VAddrDef->getOperand(1).getImm() == 0)))
+ VAddrIsZero = true;
+ }
+
+ Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
+
+ if (VAddrIsZero) {
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr)
+ .addReg(SAddrReg);
+ } else {
+ MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
+ MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
+
+ Register VAddrHiExt =
+ MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
+ .addImm(31)
+ .addReg(OldVAddrReg);
+
+ Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo)
+ .addReg(SAddrLoPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi)
+ .addReg(SAddrHiPhys);
+
+ MCRegister VCC = TRI->getVCC();
+
+ auto AddLo =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), NewVAddr)
+ .addDef(VCC)
+ .addReg(SAddrLo)
+ .addReg(OldVAddrReg)
+ .addImm(0);
+ AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
+ AddLo->getOperand(0).setIsUndef(true);
+
+ auto AddHi =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
+ .addDef(VCC, RegState::Dead)
+ .addReg(SAddrHi)
+ .addReg(VAddrHiExt)
+ .addReg(VCC, RegState::Kill)
+ .addImm(0);
+ AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
+ }
+
+ MI.setDesc(TII->get(NewOpc));
+ VAddr.setReg(NewVAddr);
+ MI.removeOperand(OldSAddrIdx);
+
+ LLVM_DEBUG(dbgs() << " Converted to VADDR: " << MI);
+ NewAddrReg = NewVAddr;
+ return true;
+}
+
+bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ TII = ST.getInstrInfo();
+ TRI = ST.getRegisterInfo();
+ MRI = &MF.getRegInfo();
+
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : MF) {
+ SmallVector<Register, 32> AddrRegs;
+ MachineInstr *LastConverted = nullptr;
+
+ for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
+ MachineInstr &MI = *MII;
+ unsigned Opc = MI.getOpcode();
+
+ int SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0)
+ continue;
+
+ if (!TII->isFLATGlobal(MI))
+ continue;
+
+ MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ if (!SAddr.isReg() || !SAddr.getReg().isPhysical())
+ continue;
+
+ if (!TRI->isSGPRReg(*MRI, SAddr.getReg()))
+ continue;
+
+ if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
+ continue;
+
+ LLVM_DEBUG(dbgs() << "SAddr redef in window: " << MI);
+ Register NewAddr;
+ if (convertToVAddr(MI, MBB, NewAddr)) {
+ AddrRegs.push_back(NewAddr);
+ LastConverted = &MI;
+ Changed = true;
+ }
+ }
+
+ // Insert KILL after the last converted load to extend the live ranges of
+ // all address VGPRs. This prevents the register allocator from reusing
+ // the same physical registers, which would introduce WAR hazards and
+ // require s_wait_xcnt between consecutive loads.
+ if (LastConverted && AddrRegs.size() > 1) {
+ auto InsertPt = std::next(LastConverted->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, LastConverted->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : AddrRegs)
+ KillMI.addReg(R);
+ }
+ }
+
+ return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
new file mode 100644
index 0000000000000..b89815683dfc5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
@@ -0,0 +1,25 @@
+//===- SIGlobalLoadSAddrToVAddr.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#define LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class SIGlobalLoadSAddrToVAddrPass
+ : public PassInfoMixin<SIGlobalLoadSAddrToVAddrPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
new file mode 100644
index 0000000000000..574777309b1f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -0,0 +1,200 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=1 -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: env AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR=1 llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+
+# Test: Basic conversion - saddr redefined 1 instruction after load
+---
+name: basic_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: basic_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: basic_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: No conversion - saddr not redefined within window (5 nops > window=4)
+---
+name: no_redef_no_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: no_redef_no_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ;
+ ; DISABLED-LABEL: name: no_redef_no_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Window boundary - redef at exactly instruction 4 within window -> converted
+---
+name: window_boundary_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: window_boundary_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: window_boundary_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Store conversion
+---
+name: store_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: store_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0
+ ;
+ ; DISABLED-LABEL: name: store_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD_SADDR %0, %1, $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0
+...
+
+# Test: Non-zero vaddr offset requires full add sequence
+---
+name: nonzero_vaddr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45, $vgpr10
+
+ ; CONVERT-LABEL: name: nonzero_vaddr
+ ; CONVERT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
+ ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
+ ; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
+ ; CONVERT-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
+ ; CONVERT-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: nonzero_vaddr
+ ; DISABLED: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = COPY $vgpr10
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Multiple converted loads get a KILL to extend address live ranges
+---
+name: multi_load_kill
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: multi_load_kill
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
+ ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: KILL [[COPY]], [[COPY1]]
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ ;
+ ; DISABLED-LABEL: name: multi_load_kill
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 1
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ $sgpr45 = S_MOV_B32 0
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 1
+ S_ENDPGM 0, implicit %1, implicit %2
+...
>From cf335ac87d9966612e710151dc9bf45a0eed6aed Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 1 Apr 2026 08:26:06 -0500
Subject: [PATCH 2/8] isel change
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 11 +++++++
.../CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll | 29 +++++++++++++++++++
2 files changed, 40 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 598ab6f7354f5..a0d327e83166a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2103,6 +2103,17 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
isa<ConstantSDNode>(Addr))
return false;
+ // On targets with XCNT, avoid placing a pure-uniform address into SADDR
+ // with a zero voffset. When the RA reuses the SGPR pair across multiple
+ // loads (common under high SGPR pressure), SIInsertWaitcnts must serialize
+ // each load with s_wait_xcnt for XNACK replay safety. Falling through to
+ // the VADDR pattern lets addresses stay in VGPRs where the RA naturally
+ // assigns distinct registers, avoiding the hazard entirely.
+ // The add(sgpr, extend(vgpr)) patterns matched above still select SADDR
+ // since those have a stable SGPR base that is not recomputed per load.
+ if (Subtarget->hasWaitXcnt())
+ return false;
+
// It's cheaper to materialize a single 32-bit zero for vaddr than the two
// moves required to copy a 64-bit SGPR to VGPR.
SAddr = Addr;
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
new file mode 100644
index 0000000000000..3b46486332ca8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+
+; On GFX1250 (hasWaitXcnt), pure-uniform addresses should use VADDR to
+; avoid s_wait_xcnt serialization when the RA reuses the SGPR pair.
+; On GFX12, SADDR is still preferred for pure-uniform addresses.
+
+define amdgpu_ps float @pure_uniform_global_load(ptr addrspace(1) inreg %sbase) {
+; GFX1250-LABEL: pure_uniform_global_load:
+; GFX1250: global_load_b32 v0, v[0:1], off
+;
+; GFX12-LABEL: pure_uniform_global_load:
+; GFX12: global_load_b32 v0, v0, s[2:3]
+ %val = load volatile float, ptr addrspace(1) %sbase
+ ret float %val
+}
+
+; Verify that add(sgpr, zext(vgpr)) still selects SADDR on GFX1250.
+define amdgpu_ps float @sgpr_plus_vgpr_offset(ptr addrspace(1) inreg %sbase, i32 %voffset) {
+; GFX1250-LABEL: sgpr_plus_vgpr_offset:
+; GFX1250: global_load_b32 v0, v0, s[2:3]
+;
+; GFX12-LABEL: sgpr_plus_vgpr_offset:
+; GFX12: global_load_b32 v0, v0, s[2:3]
+ %zext = zext i32 %voffset to i64
+ %gep = getelementptr i8, ptr addrspace(1) %sbase, i64 %zext
+ %val = load volatile float, ptr addrspace(1) %gep
+ ret float %val
+}
>From ea277712dd78c766b60f4e835307120bfe48d5a6 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Fri, 3 Apr 2026 13:18:55 -0500
Subject: [PATCH 3/8] Revert "isel change"
This reverts commit cf335ac87d9966612e710151dc9bf45a0eed6aed.
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 11 -------
.../CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll | 29 -------------------
2 files changed, 40 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a0d327e83166a..598ab6f7354f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2103,17 +2103,6 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
isa<ConstantSDNode>(Addr))
return false;
- // On targets with XCNT, avoid placing a pure-uniform address into SADDR
- // with a zero voffset. When the RA reuses the SGPR pair across multiple
- // loads (common under high SGPR pressure), SIInsertWaitcnts must serialize
- // each load with s_wait_xcnt for XNACK replay safety. Falling through to
- // the VADDR pattern lets addresses stay in VGPRs where the RA naturally
- // assigns distinct registers, avoiding the hazard entirely.
- // The add(sgpr, extend(vgpr)) patterns matched above still select SADDR
- // since those have a stable SGPR base that is not recomputed per load.
- if (Subtarget->hasWaitXcnt())
- return false;
-
// It's cheaper to materialize a single 32-bit zero for vaddr than the two
// moves required to copy a 64-bit SGPR to VGPR.
SAddr = Addr;
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
deleted file mode 100644
index 3b46486332ca8..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-
-; On GFX1250 (hasWaitXcnt), pure-uniform addresses should use VADDR to
-; avoid s_wait_xcnt serialization when the RA reuses the SGPR pair.
-; On GFX12, SADDR is still preferred for pure-uniform addresses.
-
-define amdgpu_ps float @pure_uniform_global_load(ptr addrspace(1) inreg %sbase) {
-; GFX1250-LABEL: pure_uniform_global_load:
-; GFX1250: global_load_b32 v0, v[0:1], off
-;
-; GFX12-LABEL: pure_uniform_global_load:
-; GFX12: global_load_b32 v0, v0, s[2:3]
- %val = load volatile float, ptr addrspace(1) %sbase
- ret float %val
-}
-
-; Verify that add(sgpr, zext(vgpr)) still selects SADDR on GFX1250.
-define amdgpu_ps float @sgpr_plus_vgpr_offset(ptr addrspace(1) inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: sgpr_plus_vgpr_offset:
-; GFX1250: global_load_b32 v0, v0, s[2:3]
-;
-; GFX12-LABEL: sgpr_plus_vgpr_offset:
-; GFX12: global_load_b32 v0, v0, s[2:3]
- %zext = zext i32 %voffset to i64
- %gep = getelementptr i8, ptr addrspace(1) %sbase, i64 %zext
- %val = load volatile float, ptr addrspace(1) %gep
- ret float %val
-}
>From 91b62dbd8b9d2ff44e4201ddf21bf3bc569d28c5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Fri, 3 Apr 2026 13:44:40 -0500
Subject: [PATCH 4/8] new_changes
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 78 +++++++++++-----
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 6 +-
.../wait-xcnt-atomic-rmw-optimization.ll | 89 ++++++++++---------
.../waitcnt-loop-ds-prefetch-pattern.ll | 29 +++---
4 files changed, 121 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 838b22132943f..0d3090c478b5c 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -17,6 +17,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -40,12 +41,15 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
cl::Hidden);
-static bool isEnabled() {
+static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
+ "amdgpu-global-load-saddr-to-vaddr-kill-group-size",
+ cl::desc("Max converted loads per KILL group (0 = single KILL per block)"),
+ cl::init(16), cl::Hidden);
+
+static bool isEnabled(const GCNSubtarget &ST) {
if (EnableSAddrToVAddr.getNumOccurrences())
return EnableSAddrToVAddr;
- if (const char *Env = std::getenv("AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR"))
- return StringRef(Env) != "0";
- return false;
+ return ST.hasWaitXcnt();
}
namespace {
@@ -54,6 +58,7 @@ class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
@@ -70,7 +75,8 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (!isEnabled())
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!isEnabled(ST))
return false;
SIGlobalLoadSAddrToVAddr Impl;
return Impl.run(MF);
@@ -99,7 +105,8 @@ char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
PreservedAnalyses
SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!isEnabled())
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!isEnabled(ST))
return PreservedAnalyses::all();
SIGlobalLoadSAddrToVAddr Impl;
if (!Impl.run(MF))
@@ -221,11 +228,37 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ unsigned Occupancy = MFI->getOccupancy();
+ unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+
+ // Reserve at most a quarter of the VGPR budget for converted address
+ // registers. Each conversion creates one vreg_64 kept live to the
+ // group's KILL, so the cost is 2 VGPRs per conversion.
+ MaxAddrVGPRsPerBB = MaxVGPRs / 4;
+
+ LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
+ << " maxVGPRs=" << MaxVGPRs
+ << " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
+
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
- SmallVector<Register, 32> AddrRegs;
- MachineInstr *LastConverted = nullptr;
+ SmallVector<Register, 8> GroupRegs;
+ MachineInstr *GroupLast = nullptr;
+ unsigned TotalAddrVGPRs = 0;
+
+ auto FlushGroup = [&]() {
+ if (GroupLast && GroupRegs.size() > 1) {
+ auto InsertPt = std::next(GroupLast->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : GroupRegs)
+ KillMI.addReg(R);
+ }
+ GroupRegs.clear();
+ GroupLast = nullptr;
+ };
for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
MachineInstr &MI = *MII;
@@ -248,26 +281,27 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
continue;
- LLVM_DEBUG(dbgs() << "SAddr redef in window: " << MI);
+ // Stop converting in this BB if we would exceed the VGPR budget.
+ if (TotalAddrVGPRs + 2 > MaxAddrVGPRsPerBB) {
+ LLVM_DEBUG(dbgs() << " Skipping (VGPR budget exhausted): " << MI);
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
Register NewAddr;
if (convertToVAddr(MI, MBB, NewAddr)) {
- AddrRegs.push_back(NewAddr);
- LastConverted = &MI;
+ GroupRegs.push_back(NewAddr);
+ GroupLast = &MI;
+ TotalAddrVGPRs += 2;
Changed = true;
+
+ if (SAddrToVAddrKillGroupSize > 0 &&
+ GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ FlushGroup();
}
}
- // Insert KILL after the last converted load to extend the live ranges of
- // all address VGPRs. This prevents the register allocator from reusing
- // the same physical registers, which would introduce WAR hazards and
- // require s_wait_xcnt between consecutive loads.
- if (LastConverted && AddrRegs.size() > 1) {
- auto InsertPt = std::next(LastConverted->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, LastConverted->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
- for (Register R : AddrRegs)
- KillMI.addReg(R);
- }
+ FlushGroup();
}
return Changed;
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 574777309b1f0..2d94c4c1c67f3 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=1 -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
-# RUN: env AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR=1 llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=0 -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
# Test: Basic conversion - saddr redefined 1 instruction after load
---
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 0f6edafb22908..0c29a1ee71071 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -965,57 +965,60 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
+; GFX1250-NEXT: v_writelane_b32 v5, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s3, 1
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s1, 2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v5, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: .LBB17_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 3
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 2
-; GFX1250-NEXT: v_readlane_b32 s2, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s3, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s2, 1
; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB17_1
@@ -1047,48 +1050,51 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
-; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v5, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: v_mov_b32_e32 v2, 1
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: v_mov_b32_e32 v4, 2
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s0, -1
; GFX1250-NEXT: s_mov_b32 s2, 0
; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
; GFX1250-NEXT: .LBB18_1: ; %Flow
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 2
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 2
; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX1250-NEXT: s_mov_b32 s0, 1
; GFX1250-NEXT: v_cmp_ne_u32_e64 s0, v0, s0
@@ -1096,12 +1102,12 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
; GFX1250-NEXT: ; %bb.2: ; %bb1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 3
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1117,39 +1123,40 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_branch .LBB18_4
; GFX1250-NEXT: .LBB18_3: ; %bb2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: v_mov_b32_e32 v2, 4
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_branch .LBB18_1
; GFX1250-NEXT: .LBB18_4: ; %merge
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 5
; GFX1250-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index 7f98f8cf8de06..af53cb587bc7d 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -12,14 +12,14 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
; CHECK-NEXT: v_and_b32_e32 v12, 0x3ff, v0
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v6, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, v4
+; CHECK-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8
+; CHECK-NEXT: v_mov_b32_e32 v11, v8
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s1
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: ds_load_b128 v[8:11], v13
+; CHECK-NEXT: ds_load_b128 v[4:7], v13
; CHECK-NEXT: ds_load_b128 v[0:3], v13 offset:16
; CHECK-NEXT: s_wait_dscnt 0x0
; CHECK-NEXT: .LBB0_1: ; %loop
@@ -27,24 +27,23 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_barrier_signal -1
; CHECK-NEXT: s_add_co_i32 s1, s1, 1
; CHECK-NEXT: s_wait_dscnt 0x1
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[10:11]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[8:9]
-; CHECK-NEXT: v_lshl_add_u32 v14, s1, 5, v13
+; CHECK-NEXT: v_pk_add_f32 v[6:7], v[10:11], v[6:7]
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[8:9], v[4:5]
+; CHECK-NEXT: v_lshl_add_u32 v10, s1, 5, v13
; CHECK-NEXT: s_cmp_lt_i32 s1, s0
; CHECK-NEXT: s_wait_dscnt 0x0
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[2:3]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[0:1]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[6:7]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[4:5]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], v[6:7], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[14:15], v[4:5], v[0:1]
; CHECK-NEXT: s_barrier_wait -1
-; CHECK-NEXT: ds_load_b128 v[8:11], v14
-; CHECK-NEXT: ds_load_b128 v[0:3], v14 offset:16
+; CHECK-NEXT: ds_load_b128 v[4:7], v10
+; CHECK-NEXT: ds_load_b128 v[0:3], v10 offset:16
+; CHECK-NEXT: v_pk_add_f32 v[10:11], v[8:9], v[8:9]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], v[14:15], v[14:15]
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: global_store_b128 v12, v[4:7], s[0:1] scale_offset
+; CHECK-NEXT: global_store_b128 v12, v[8:11], s[0:1] scale_offset
; CHECK-NEXT: s_endpgm
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
>From c8536d667046dd894e8dccb6452b5a135760a3b5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Sat, 4 Apr 2026 10:10:58 -0500
Subject: [PATCH 5/8] saddr
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 286 +++++++++++-------
.../CodeGen/AMDGPU/saddr-to-vaddr-budget.mir | 52 ++++
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 82 +++--
3 files changed, 260 insertions(+), 160 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 0d3090c478b5c..802425daf7bc8 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -1,4 +1,4 @@
-//===-- SIGlobalLoadSAddrToVAddr.cpp ---------------------------------------===//
+//===- SIGlobalLoadSAddrToVAddr.cpp - SADDR global loads to VADDR ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,9 +6,9 @@
//
//===----------------------------------------------------------------------===//
//
-// Convert SADDR global loads/stores to VADDR form when the SGPR address pair
-// is about to be overwritten. This eliminates s_wait_xcnt hazards that
-// serialize loads sharing a single SGPR scratch address register.
+/// \file Convertglobal SADDR memory ops to VADDR when the SGPR address is
+/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
+/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills;
//
//===----------------------------------------------------------------------===//
@@ -19,8 +19,11 @@
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -30,12 +33,6 @@ using namespace llvm;
#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
-static cl::opt<bool> EnableSAddrToVAddr(
- "amdgpu-global-load-saddr-to-vaddr",
- cl::desc(
- "Convert SADDR global loads/stores to VADDR when saddr is overwritten"),
- cl::Hidden);
-
static cl::opt<unsigned> SAddrToVAddrWindow(
"amdgpu-global-load-saddr-to-vaddr-window",
cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
@@ -43,26 +40,39 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
"amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group (0 = single KILL per block)"),
+ cl::desc("Max converted loads per KILL group (0 disables KILL insertion)"),
cl::init(16), cl::Hidden);
-static bool isEnabled(const GCNSubtarget &ST) {
- if (EnableSAddrToVAddr.getNumOccurrences())
- return EnableSAddrToVAddr;
- return ST.hasWaitXcnt();
-}
+static cl::opt<unsigned> SAddrToVAddrMaxConverts(
+ "amdgpu-global-load-saddr-to-vaddr-max-converts",
+ cl::desc(
+ "Override max conversions per BB (0 = use occupancy-based budget)"),
+ cl::init(0), cl::Hidden);
namespace {
+struct Candidate {
+ MachineInstr *MI;
+ unsigned Pressure;
+};
+
class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ MachineFunction *CurMF = nullptr;
unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
- bool sAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+ bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+
+ unsigned countLiveSGPR32(const LiveRegUnits &LRU) const;
+
+ void computeSGPRPressure(MachineBasicBlock &MBB,
+ DenseMap<MachineInstr *, unsigned> &Out);
+
+ bool processMBB(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF);
@@ -75,8 +85,7 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!isEnabled(ST))
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
SIGlobalLoadSAddrToVAddr Impl;
return Impl.run(MF);
@@ -105,8 +114,7 @@ char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
PreservedAnalyses
SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!isEnabled(ST))
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
SIGlobalLoadSAddrToVAddr Impl;
if (!Impl.run(MF))
@@ -116,26 +124,66 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
return PA;
}
-bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedInWindow(MachineInstr &MI,
- Register SAddr) {
+unsigned
+SIGlobalLoadSAddrToVAddr::countLiveSGPR32(const LiveRegUnits &LRU) const {
unsigned Count = 0;
- MachineBasicBlock::iterator Next = std::next(MI.getIterator());
- MachineBasicBlock::iterator End = MI.getParent()->end();
- for (; Next != End && Count < SAddrToVAddrWindow; ++Next) {
- if (Next->isMetaInstruction())
+ for (MCPhysReg Reg : TRI->getAllSGPR32(*CurMF)) {
+ if (!LRU.available(Reg))
+ ++Count;
+ }
+ return Count;
+}
+
+void SIGlobalLoadSAddrToVAddr::computeSGPRPressure(
+ MachineBasicBlock &MBB, DenseMap<MachineInstr *, unsigned> &Out) {
+ Out.clear();
+ LiveRegUnits Live(*TRI);
+ Live.addLiveOuts(MBB);
+
+ for (MachineInstr &MI : reverse(MBB)) {
+ Live.stepBackward(MI);
+ Out[&MI] = countLiveSGPR32(Live);
+ }
+}
+
+bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
+ Register SAddr) {
+ unsigned NumScanned = 0;
+ auto I = std::next(MI.getIterator());
+ auto E = MI.getParent()->end();
+ for (; I != E; ++I) {
+ if (I->isMetaInstruction())
continue;
- ++Count;
- for (const MachineOperand &MO : Next->operands()) {
- if (!MO.isReg() || !MO.isDef())
- continue;
- Register Reg = MO.getReg();
- if (Reg.isPhysical() && TRI->regsOverlap(Reg, SAddr))
+
+ ++NumScanned;
+
+ for (const MachineOperand &MO : I->operands()) {
+ if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
+ TRI->regsOverlap(MO.getReg(), SAddr))
return true;
}
+
+ if (NumScanned >= SAddrToVAddrWindow)
+ return false;
}
return false;
}
+/// Return true if \p MI is a FLAT global memory op with a physical SGPR saddr.
+static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
+ const SIInstrInfo &TII,
+ const SIRegisterInfo &TRI,
+ const MachineRegisterInfo &MRI,
+ int &SAddrIdx) {
+ unsigned Opc = MI.getOpcode();
+ SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ return false;
+ const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ return SAddr.isReg() && SAddr.getReg().isPhysical() &&
+ TRI.isSGPRReg(MRI, SAddr.getReg());
+}
+
bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
@@ -144,8 +192,7 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
if (NewOpc < 0)
return false;
- int NewVAddrIdx =
- AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
+ int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
if (NewVAddrIdx < 0)
return false;
@@ -157,7 +204,6 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineOperand &VAddr = MI.getOperand(OldVAddrIdx);
Register SAddrReg = SAddr.getReg();
Register OldVAddrReg = VAddr.getReg();
-
const DebugLoc &DL = MI.getDebugLoc();
bool VAddrIsZero = false;
@@ -173,24 +219,24 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
if (VAddrIsZero) {
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr)
- .addReg(SAddrReg);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
} else {
+ assert(OldVAddrReg.isVirtual() &&
+ TRI->getRegSizeInBits(*MRI->getRegClass(OldVAddrReg)) == 32 &&
+ "Non-zero vaddr path expects a 32-bit VGPR operand");
+
MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
- Register VAddrHiExt =
- MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register VAddrHiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
.addImm(31)
.addReg(OldVAddrReg);
Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo)
- .addReg(SAddrLoPhys);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi)
- .addReg(SAddrHiPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo).addReg(SAddrLoPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi).addReg(SAddrHiPhys);
MCRegister VCC = TRI->getVCC();
@@ -222,87 +268,101 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
return true;
}
+bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
+ SmallVector<MachineInstr *, 16> EligibleMIs;
+ for (MachineInstr &MI : MBB) {
+ int SAddrIdx;
+ if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ continue;
+ if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
+ continue;
+ EligibleMIs.push_back(&MI);
+ }
+
+ if (EligibleMIs.empty())
+ return false;
+
+ DenseMap<MachineInstr *, unsigned> PressureMap;
+ computeSGPRPressure(MBB, PressureMap);
+
+ SmallVector<Candidate, 16> Candidates;
+ for (MachineInstr *MI : EligibleMIs)
+ Candidates.push_back({MI, PressureMap.lookup(MI)});
+
+ // Prefer highest SGPR pressure; each conversion costs one vreg_64.
+ const unsigned MaxConverts = SAddrToVAddrMaxConverts > 0
+ ? SAddrToVAddrMaxConverts.getValue()
+ : MaxAddrVGPRsPerBB / 2;
+ DenseSet<MachineInstr *> Allowed;
+ llvm::sort(Candidates, [](const Candidate &A, const Candidate &B) {
+ return A.Pressure > B.Pressure;
+ });
+ const unsigned NumToConvert =
+ std::min(static_cast<unsigned>(Candidates.size()), MaxConverts);
+ for (unsigned I = 0; I < NumToConvert; ++I)
+ Allowed.insert(Candidates[I].MI);
+
+ SmallVector<Register, 8> GroupRegs;
+ MachineInstr *GroupLast = nullptr;
+ bool Changed = false;
+
+ // KILL extends address VGPR liveness to prevent RA reuse within a group.
+ auto FlushGroup = [&]() {
+ if (GroupLast && GroupRegs.size() > 1) {
+ auto InsertPt = std::next(GroupLast->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : GroupRegs)
+ KillMI.addReg(R, RegState::Kill);
+ }
+ GroupRegs.clear();
+ GroupLast = nullptr;
+ };
+
+ for (MachineInstr &MI : MBB) {
+ if (!Allowed.count(&MI))
+ continue;
+
+ LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
+ Register NewAddr;
+ if (convertToVAddr(MI, MBB, NewAddr)) {
+ GroupRegs.push_back(NewAddr);
+ GroupLast = &MI;
+ Changed = true;
+
+ if (SAddrToVAddrKillGroupSize > 0 &&
+ GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ FlushGroup();
+ }
+ }
+
+ FlushGroup();
+ return Changed;
+}
+
bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
+ CurMF = &MF;
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
unsigned Occupancy = MFI->getOccupancy();
- unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+ unsigned DynBlockSize = MFI->getDynamicVGPRBlockSize();
+ unsigned VGPRsAtOccupancy = ST.getMaxNumVGPRs(Occupancy, DynBlockSize);
- // Reserve at most a quarter of the VGPR budget for converted address
- // registers. Each conversion creates one vreg_64 kept live to the
- // group's KILL, so the cost is 2 VGPRs per conversion.
- MaxAddrVGPRsPerBB = MaxVGPRs / 4;
+ // Cap address VGPRs per BB to half the per-wave budget at target occupancy.
+ MaxAddrVGPRsPerBB = VGPRsAtOccupancy / 2;
LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
- << " maxVGPRs=" << MaxVGPRs
+ << " VGPRsAtOcc=" << VGPRsAtOccupancy
<< " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
bool Changed = false;
-
- for (MachineBasicBlock &MBB : MF) {
- SmallVector<Register, 8> GroupRegs;
- MachineInstr *GroupLast = nullptr;
- unsigned TotalAddrVGPRs = 0;
-
- auto FlushGroup = [&]() {
- if (GroupLast && GroupRegs.size() > 1) {
- auto InsertPt = std::next(GroupLast->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
- for (Register R : GroupRegs)
- KillMI.addReg(R);
- }
- GroupRegs.clear();
- GroupLast = nullptr;
- };
-
- for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
- MachineInstr &MI = *MII;
- unsigned Opc = MI.getOpcode();
-
- int SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
- if (SAddrIdx < 0)
- continue;
-
- if (!TII->isFLATGlobal(MI))
- continue;
-
- MachineOperand &SAddr = MI.getOperand(SAddrIdx);
- if (!SAddr.isReg() || !SAddr.getReg().isPhysical())
- continue;
-
- if (!TRI->isSGPRReg(*MRI, SAddr.getReg()))
- continue;
-
- if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
- continue;
-
- // Stop converting in this BB if we would exceed the VGPR budget.
- if (TotalAddrVGPRs + 2 > MaxAddrVGPRsPerBB) {
- LLVM_DEBUG(dbgs() << " Skipping (VGPR budget exhausted): " << MI);
- continue;
- }
-
- LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
- Register NewAddr;
- if (convertToVAddr(MI, MBB, NewAddr)) {
- GroupRegs.push_back(NewAddr);
- GroupLast = &MI;
- TotalAddrVGPRs += 2;
- Changed = true;
-
- if (SAddrToVAddrKillGroupSize > 0 &&
- GroupRegs.size() >= SAddrToVAddrKillGroupSize)
- FlushGroup();
- }
- }
-
- FlushGroup();
- }
+ for (MachineBasicBlock &MBB : MF)
+ Changed |= processMBB(MBB);
return Changed;
}
+
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
new file mode 100644
index 0000000000000..8f5c706d7504e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
+# RUN: -amdgpu-global-load-saddr-to-vaddr-max-converts=1 -o - %s \
+# RUN: | FileCheck -check-prefix=CAP1 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
+# RUN: -o - %s | FileCheck -check-prefix=NOBUDGET %s
+
+# Test: With max-converts=1, only the highest SGPR-pressure candidate is
+# converted. Candidate 1 has SGPR pressure 4 (sgpr44, sgpr45, sgpr46, sgpr47
+# all live), candidate 2 has pressure 2 (only sgpr44, sgpr45 live).
+# The pass picks candidate 1 for conversion and leaves candidate 2 as SADDR.
+# Without the budget cap, both are converted.
+---
+name: budget_cap_picks_highest_pressure
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+
+ ; CAP1-LABEL: name: budget_cap_picks_highest_pressure
+ ; CAP1: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+ ; CAP1-NEXT: {{ $}}
+ ; CAP1-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CAP1-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CAP1-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
+ ; CAP1-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
+ ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CAP1-NEXT: $sgpr44 = S_MOV_B32 1
+ ; CAP1-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ;
+ ; NOBUDGET-LABEL: name: budget_cap_picks_highest_pressure
+ ; NOBUDGET: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+ ; NOBUDGET-NEXT: {{ $}}
+ ; NOBUDGET-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; NOBUDGET-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
+ ; NOBUDGET-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
+ ; NOBUDGET-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; NOBUDGET-NEXT: KILL killed [[COPY]], killed [[COPY1]]
+ ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 1
+ ; NOBUDGET-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 $sgpr46
+ $sgpr45 = S_MOV_B32 $sgpr47
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 1
+ S_ENDPGM 0, implicit %1, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 2d94c4c1c67f3..f05d7968c3838 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,5 +1,5 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=0 -o - %s | FileCheck -check-prefix=DISABLED %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
# Test: Basic conversion - saddr redefined 1 instruction after load
@@ -11,14 +11,18 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: basic_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
; DISABLED-LABEL: name: basic_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
@@ -37,7 +41,9 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: no_redef_no_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
@@ -48,7 +54,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
;
; DISABLED-LABEL: name: no_redef_no_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: S_NOP 0
; DISABLED-NEXT: S_NOP 0
@@ -68,41 +76,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Window boundary - redef at exactly instruction 4 within window -> converted
----
-name: window_boundary_convert
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45
-
- ; CONVERT-LABEL: name: window_boundary_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
- ;
- ; DISABLED-LABEL: name: window_boundary_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- S_NOP 0
- S_NOP 0
- S_NOP 0
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
-...
-
# Test: Store conversion
---
name: store_convert
@@ -112,7 +85,9 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: store_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
@@ -120,7 +95,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0
;
; DISABLED-LABEL: name: store_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
@@ -141,7 +118,9 @@ body: |
liveins: $sgpr44_sgpr45, $vgpr10
; CONVERT-LABEL: name: nonzero_vaddr
- ; CONVERT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; CONVERT: liveins: $sgpr44_sgpr45, $vgpr10
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
@@ -152,7 +131,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
; DISABLED-LABEL: name: nonzero_vaddr
- ; DISABLED: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DISABLED: liveins: $sgpr44_sgpr45, $vgpr10
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
@@ -171,19 +152,23 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: multi_load_kill
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: KILL [[COPY]], [[COPY1]]
+ ; CONVERT-NEXT: KILL killed [[COPY]], killed [[COPY1]]
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
; DISABLED-LABEL: name: multi_load_kill
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
@@ -198,3 +183,6 @@ body: |
$sgpr44 = S_MOV_B32 1
S_ENDPGM 0, implicit %1, implicit %2
...
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# CONVERT: {{.*}}
+# DISABLED: {{.*}}
>From 58ff4cbe4af29f3490db35b345afe46801d8ae63 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Sat, 4 Apr 2026 10:36:24 -0500
Subject: [PATCH 6/8] removed budget
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 107 ++----------------
.../CodeGen/AMDGPU/saddr-to-vaddr-budget.mir | 52 ---------
2 files changed, 7 insertions(+), 152 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 802425daf7bc8..64212ede8b3c2 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -6,9 +6,9 @@
//
//===----------------------------------------------------------------------===//
//
-/// \file Convertglobal SADDR memory ops to VADDR when the SGPR address is
+/// \file Convert global SADDR memory ops to VADDR when the SGPR address is
/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
-/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills;
+/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills.
//
//===----------------------------------------------------------------------===//
@@ -17,13 +17,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
-#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -40,38 +34,18 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
"amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group (0 disables KILL insertion)"),
- cl::init(16), cl::Hidden);
-
-static cl::opt<unsigned> SAddrToVAddrMaxConverts(
- "amdgpu-global-load-saddr-to-vaddr-max-converts",
- cl::desc(
- "Override max conversions per BB (0 = use occupancy-based budget)"),
- cl::init(0), cl::Hidden);
+ cl::desc("Max converted loads per KILL group"), cl::init(16), cl::Hidden);
namespace {
-struct Candidate {
- MachineInstr *MI;
- unsigned Pressure;
-};
-
class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
- MachineFunction *CurMF = nullptr;
- unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
-
- unsigned countLiveSGPR32(const LiveRegUnits &LRU) const;
-
- void computeSGPRPressure(MachineBasicBlock &MBB,
- DenseMap<MachineInstr *, unsigned> &Out);
-
bool processMBB(MachineBasicBlock &MBB);
public:
@@ -124,28 +98,6 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
return PA;
}
-unsigned
-SIGlobalLoadSAddrToVAddr::countLiveSGPR32(const LiveRegUnits &LRU) const {
- unsigned Count = 0;
- for (MCPhysReg Reg : TRI->getAllSGPR32(*CurMF)) {
- if (!LRU.available(Reg))
- ++Count;
- }
- return Count;
-}
-
-void SIGlobalLoadSAddrToVAddr::computeSGPRPressure(
- MachineBasicBlock &MBB, DenseMap<MachineInstr *, unsigned> &Out) {
- Out.clear();
- LiveRegUnits Live(*TRI);
- Live.addLiveOuts(MBB);
-
- for (MachineInstr &MI : reverse(MBB)) {
- Live.stepBackward(MI);
- Out[&MI] = countLiveSGPR32(Live);
- }
-}
-
bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
Register SAddr) {
unsigned NumScanned = 0;
@@ -169,7 +121,6 @@ bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
return false;
}
-/// Return true if \p MI is a FLAT global memory op with a physical SGPR saddr.
static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
const SIInstrInfo &TII,
const SIRegisterInfo &TRI,
@@ -269,39 +220,6 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
}
bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
- SmallVector<MachineInstr *, 16> EligibleMIs;
- for (MachineInstr &MI : MBB) {
- int SAddrIdx;
- if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
- continue;
- if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
- continue;
- EligibleMIs.push_back(&MI);
- }
-
- if (EligibleMIs.empty())
- return false;
-
- DenseMap<MachineInstr *, unsigned> PressureMap;
- computeSGPRPressure(MBB, PressureMap);
-
- SmallVector<Candidate, 16> Candidates;
- for (MachineInstr *MI : EligibleMIs)
- Candidates.push_back({MI, PressureMap.lookup(MI)});
-
- // Prefer highest SGPR pressure; each conversion costs one vreg_64.
- const unsigned MaxConverts = SAddrToVAddrMaxConverts > 0
- ? SAddrToVAddrMaxConverts.getValue()
- : MaxAddrVGPRsPerBB / 2;
- DenseSet<MachineInstr *> Allowed;
- llvm::sort(Candidates, [](const Candidate &A, const Candidate &B) {
- return A.Pressure > B.Pressure;
- });
- const unsigned NumToConvert =
- std::min(static_cast<unsigned>(Candidates.size()), MaxConverts);
- for (unsigned I = 0; I < NumToConvert; ++I)
- Allowed.insert(Candidates[I].MI);
-
SmallVector<Register, 8> GroupRegs;
MachineInstr *GroupLast = nullptr;
bool Changed = false;
@@ -320,7 +238,10 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
};
for (MachineInstr &MI : MBB) {
- if (!Allowed.count(&MI))
+ int SAddrIdx;
+ if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ continue;
+ if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
continue;
LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
@@ -345,19 +266,6 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
- CurMF = &MF;
-
- const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
- unsigned Occupancy = MFI->getOccupancy();
- unsigned DynBlockSize = MFI->getDynamicVGPRBlockSize();
- unsigned VGPRsAtOccupancy = ST.getMaxNumVGPRs(Occupancy, DynBlockSize);
-
- // Cap address VGPRs per BB to half the per-wave budget at target occupancy.
- MaxAddrVGPRsPerBB = VGPRsAtOccupancy / 2;
-
- LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
- << " VGPRsAtOcc=" << VGPRsAtOccupancy
- << " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
@@ -365,4 +273,3 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
return Changed;
}
-
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
deleted file mode 100644
index 8f5c706d7504e..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
+++ /dev/null
@@ -1,52 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
-# RUN: -amdgpu-global-load-saddr-to-vaddr-max-converts=1 -o - %s \
-# RUN: | FileCheck -check-prefix=CAP1 %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
-# RUN: -o - %s | FileCheck -check-prefix=NOBUDGET %s
-
-# Test: With max-converts=1, only the highest SGPR-pressure candidate is
-# converted. Candidate 1 has SGPR pressure 4 (sgpr44, sgpr45, sgpr46, sgpr47
-# all live), candidate 2 has pressure 2 (only sgpr44, sgpr45 live).
-# The pass picks candidate 1 for conversion and leaves candidate 2 as SADDR.
-# Without the budget cap, both are converted.
----
-name: budget_cap_picks_highest_pressure
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
-
- ; CAP1-LABEL: name: budget_cap_picks_highest_pressure
- ; CAP1: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
- ; CAP1-NEXT: {{ $}}
- ; CAP1-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CAP1-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CAP1-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
- ; CAP1-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
- ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CAP1-NEXT: $sgpr44 = S_MOV_B32 1
- ; CAP1-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- ;
- ; NOBUDGET-LABEL: name: budget_cap_picks_highest_pressure
- ; NOBUDGET: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
- ; NOBUDGET-NEXT: {{ $}}
- ; NOBUDGET-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; NOBUDGET-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
- ; NOBUDGET-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
- ; NOBUDGET-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; NOBUDGET-NEXT: KILL killed [[COPY]], killed [[COPY1]]
- ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 1
- ; NOBUDGET-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 $sgpr46
- $sgpr45 = S_MOV_B32 $sgpr47
- %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 1
- S_ENDPGM 0, implicit %1, implicit %2
-...
>From 28b6d9c89bbdcbd2b964de0dfd0b2087945d7062 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 10:35:01 -0500
Subject: [PATCH 7/8] changes
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 148 ++++++++----------
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 20 +--
2 files changed, 76 insertions(+), 92 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 64212ede8b3c2..d33396422264d 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -5,11 +5,12 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//
-/// \file Convert global SADDR memory ops to VADDR when the SGPR address is
-/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
-/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills.
-//
+/// \file
+/// Convert FLAT global SADDR memory ops to VADDR when the physical SGPR
+/// address pair is redefined later in the block. This avoids s_wait_xcnt
+/// stalls from XNACK replay of loads that share a reused SGPR pair.
+///
+/// Runs after SGPR RA / SILowerSGPRSpills and before VGPR RA.
//===----------------------------------------------------------------------===//
#include "SIGlobalLoadSAddrToVAddr.h"
@@ -18,6 +19,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -27,14 +29,18 @@ using namespace llvm;
#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
-static cl::opt<unsigned> SAddrToVAddrWindow(
- "amdgpu-global-load-saddr-to-vaddr-window",
- cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
- cl::Hidden);
+STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
-static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
- "amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group"), cl::init(16), cl::Hidden);
+static cl::opt<unsigned> SAddrRedefWindow(
+ "amdgpu-saddr-redef-window",
+ cl::desc("Max non-meta instructions to scan for saddr redef "
+ "(0 = scan to end of block)"),
+ cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> AddrLivenessGroupSize(
+ "amdgpu-saddr-vaddr-liveness-group",
+ cl::desc("Max converted VADDR addresses kept simultaneously live"),
+ cl::init(16), cl::Hidden);
namespace {
@@ -43,9 +49,9 @@ class SIGlobalLoadSAddrToVAddr {
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ bool sAddrRedefinedAfter(MachineInstr &MI, Register SAddr);
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
- bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
bool processMBB(MachineBasicBlock &MBB);
public:
@@ -55,14 +61,12 @@ class SIGlobalLoadSAddrToVAddr {
class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
public:
static char ID;
-
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- SIGlobalLoadSAddrToVAddr Impl;
- return Impl.run(MF);
+ return SIGlobalLoadSAddrToVAddr().run(MF);
}
StringRef getPassName() const override {
@@ -90,51 +94,43 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- SIGlobalLoadSAddrToVAddr Impl;
- if (!Impl.run(MF))
+ if (!SIGlobalLoadSAddrToVAddr().run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
return PA;
}
-bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
- Register SAddr) {
+/// Return true if \p MI is a FLAT global op with a physical SGPR saddr.
+static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
+ const SIRegisterInfo &TRI,
+ const MachineRegisterInfo &MRI, int &SAddrIdx) {
+ SAddrIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ return false;
+ const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ return SAddr.isReg() && SAddr.getReg().isPhysical() &&
+ TRI.isSGPRReg(MRI, SAddr.getReg());
+}
+
+bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
+ Register SAddr) {
unsigned NumScanned = 0;
- auto I = std::next(MI.getIterator());
- auto E = MI.getParent()->end();
+ MachineBasicBlock::iterator I = std::next(MI.getIterator());
+ MachineBasicBlock::iterator E = MI.getParent()->end();
for (; I != E; ++I) {
- if (I->isMetaInstruction())
- continue;
-
- ++NumScanned;
-
- for (const MachineOperand &MO : I->operands()) {
+ for (const MachineOperand &MO : I->operands())
if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
TRI->regsOverlap(MO.getReg(), SAddr))
return true;
- }
- if (NumScanned >= SAddrToVAddrWindow)
+ if (!I->isMetaInstruction() && SAddrRedefWindow > 0 &&
+ ++NumScanned >= SAddrRedefWindow)
return false;
}
return false;
}
-static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
- const SIInstrInfo &TII,
- const SIRegisterInfo &TRI,
- const MachineRegisterInfo &MRI,
- int &SAddrIdx) {
- unsigned Opc = MI.getOpcode();
- SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
- if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
- return false;
- const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
- return SAddr.isReg() && SAddr.getReg().isPhysical() &&
- TRI.isSGPRReg(MRI, SAddr.getReg());
-}
-
bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
@@ -142,9 +138,12 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
if (NewOpc < 0)
return false;
+ if (AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr) < 0)
+ return false;
- int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
- if (NewVAddrIdx < 0)
+ // VADDR form does not support scale_offset.
+ int CPolIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::cpol);
+ if (CPolIdx >= 0 && (MI.getOperand(CPolIdx).getImm() & AMDGPU::CPol::SCAL))
return false;
int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
@@ -159,12 +158,10 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
bool VAddrIsZero = false;
if (OldVAddrReg.isVirtual()) {
- MachineInstr *VAddrDef = MRI->getVRegDef(OldVAddrReg);
- if (VAddrDef &&
- (VAddrDef->isImplicitDef() ||
- (VAddrDef->isMoveImmediate() && VAddrDef->getOperand(1).isImm() &&
- VAddrDef->getOperand(1).getImm() == 0)))
- VAddrIsZero = true;
+ if (auto *Def = MRI->getVRegDef(OldVAddrReg))
+ VAddrIsZero = Def->isImplicitDef() ||
+ (Def->isMoveImmediate() && Def->getOperand(1).isImm() &&
+ Def->getOperand(1).getImm() == 0);
}
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
@@ -172,29 +169,24 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
if (VAddrIsZero) {
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
} else {
- assert(OldVAddrReg.isVirtual() &&
- TRI->getRegSizeInBits(*MRI->getRegClass(OldVAddrReg)) == 32 &&
- "Non-zero vaddr path expects a 32-bit VGPR operand");
+ MCRegister Lo = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
+ MCRegister Hi = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
- MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
- MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
-
- Register VAddrHiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
+ Register HiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
.addImm(31)
.addReg(OldVAddrReg);
- Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo).addReg(SAddrLoPhys);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi).addReg(SAddrHiPhys);
+ Register LoV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register HiV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV).addReg(Lo);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV).addReg(Hi);
MCRegister VCC = TRI->getVCC();
-
auto AddLo =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), NewVAddr)
.addDef(VCC)
- .addReg(SAddrLo)
+ .addReg(LoV)
.addReg(OldVAddrReg)
.addImm(0);
AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
@@ -203,8 +195,8 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
auto AddHi =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
.addDef(VCC, RegState::Dead)
- .addReg(SAddrHi)
- .addReg(VAddrHiExt)
+ .addReg(HiV)
+ .addReg(HiExt)
.addReg(VCC, RegState::Kill)
.addImm(0);
AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
@@ -214,8 +206,9 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
VAddr.setReg(NewVAddr);
MI.removeOperand(OldSAddrIdx);
- LLVM_DEBUG(dbgs() << " Converted to VADDR: " << MI);
+ LLVM_DEBUG(dbgs() << " Converted: " << MI);
NewAddrReg = NewVAddr;
+ ++NumConverted;
return true;
}
@@ -224,12 +217,11 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
MachineInstr *GroupLast = nullptr;
bool Changed = false;
- // KILL extends address VGPR liveness to prevent RA reuse within a group.
auto FlushGroup = [&]() {
if (GroupLast && GroupRegs.size() > 1) {
- auto InsertPt = std::next(GroupLast->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
+ auto KillMI =
+ BuildMI(MBB, std::next(GroupLast->getIterator()),
+ GroupLast->getDebugLoc(), TII->get(TargetOpcode::KILL));
for (Register R : GroupRegs)
KillMI.addReg(R, RegState::Kill);
}
@@ -239,20 +231,19 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
for (MachineInstr &MI : MBB) {
int SAddrIdx;
- if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
continue;
- if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
+ if (!sAddrRedefinedAfter(MI, MI.getOperand(SAddrIdx).getReg()))
continue;
- LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
Register NewAddr;
if (convertToVAddr(MI, MBB, NewAddr)) {
GroupRegs.push_back(NewAddr);
GroupLast = &MI;
Changed = true;
- if (SAddrToVAddrKillGroupSize > 0 &&
- GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ if (AddrLivenessGroupSize > 0 &&
+ GroupRegs.size() >= AddrLivenessGroupSize)
FlushGroup();
}
}
@@ -270,6 +261,5 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
Changed |= processMBB(MBB);
-
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index f05d7968c3838..54b52520fe91e 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -2,7 +2,6 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
-# Test: Basic conversion - saddr redefined 1 instruction after load
---
name: basic_convert
tracksRegLiveness: true
@@ -32,28 +31,29 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: No conversion - saddr not redefined within window (5 nops > window=4)
+# Default window=0 scans to end of block; redef after 5 NOPs still triggers.
---
-name: no_redef_no_convert
+name: redef_after_nops
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: no_redef_no_convert
+ ; CONVERT-LABEL: name: redef_after_nops
; CONVERT: liveins: $sgpr44_sgpr45
; CONVERT-NEXT: {{ $}}
; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: no_redef_no_convert
+ ; DISABLED-LABEL: name: redef_after_nops
; DISABLED: liveins: $sgpr44_sgpr45
; DISABLED-NEXT: {{ $}}
; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -76,7 +76,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Store conversion
---
name: store_convert
tracksRegLiveness: true
@@ -109,7 +108,6 @@ body: |
S_ENDPGM 0
...
-# Test: Non-zero vaddr offset requires full add sequence
---
name: nonzero_vaddr
tracksRegLiveness: true
@@ -143,7 +141,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Multiple converted loads get a KILL to extend address live ranges
---
name: multi_load_kill
tracksRegLiveness: true
@@ -183,6 +180,3 @@ body: |
$sgpr44 = S_MOV_B32 1
S_ENDPGM 0, implicit %1, implicit %2
...
-## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-# CONVERT: {{.*}}
-# DISABLED: {{.*}}
>From f425c50bcd47f6d832e52167b52fea589fc2b2d5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 13:48:01 -0500
Subject: [PATCH 8/8] [AMDGPU] Rewrite SADDR memory ops to VADDR reduce xcnt
stalls
When the SGPR register allocator reuses the same physical pair across
consecutive SADDR global loads, s_wait_xcnt stalls are introduced for
XNACK replay.
Add SIFixXcntStallSAddrReuse, a post-SGPR-RA pass that converts
SADDR global memory ops to VADDR form . A KILL pseudo extends
address VGPR liveness in groups of 16 to prevent the VGPR allocator
from reintroducing the same hazard. The pass scans to the end of each
block by default and skips scale_offset loads that have no VADDR
equivalent.
Enabled by default on targets with s_wait_xcnt.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 2 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 12 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 2 +-
...VAddr.cpp => SIFixXcntStallSAddrReuse.cpp} | 45 ++--
...drToVAddr.h => SIFixXcntStallSAddrReuse.h} | 12 +-
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 201 +++++++-----------
7 files changed, 117 insertions(+), 161 deletions(-)
rename llvm/lib/Target/AMDGPU/{SIGlobalLoadSAddrToVAddr.cpp => SIFixXcntStallSAddrReuse.cpp} (86%)
rename llvm/lib/Target/AMDGPU/{SIGlobalLoadSAddrToVAddr.h => SIFixXcntStallSAddrReuse.h} (60%)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 16b3d423a45ec..4370703120abc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -214,8 +214,8 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
-void initializeSIGlobalLoadSAddrToVAddrLegacyPass(PassRegistry &);
-extern char &SIGlobalLoadSAddrToVAddrLegacyID;
+void initializeSIFixXcntStallSAddrReuseLegacyPass(PassRegistry &);
+extern char &SIFixXcntStallSAddrReuseLegacyID;
void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
extern char &SILoadStoreOptimizerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 84efd6452e008..3af2204d83281 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -133,7 +133,7 @@ MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
-MACHINE_FUNCTION_PASS("si-global-load-saddr-to-vaddr", SIGlobalLoadSAddrToVAddrPass())
+MACHINE_FUNCTION_PASS("si-fix-xcnt-stall-saddr-reuse", SIFixXcntStallSAddrReusePass())
MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 6a986a16a37eb..fc8edd52557e6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -50,9 +50,9 @@
#include "R600TargetMachine.h"
#include "SIFixSGPRCopies.h"
#include "SIFixVGPRCopies.h"
+#include "SIFixXcntStallSAddrReuse.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
-#include "SIGlobalLoadSAddrToVAddr.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -728,7 +728,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
- initializeSIGlobalLoadSAddrToVAddrLegacyPass(*PR);
+ initializeSIFixXcntStallSAddrReuseLegacyPass(*PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -1816,7 +1816,7 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1855,7 +1855,7 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -2461,7 +2461,7 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2557,7 +2557,7 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 1b193a4b581fb..eb11087991ce4 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -161,7 +161,7 @@ add_llvm_target(AMDGPUCodeGen
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
SIFrameLowering.cpp
- SIGlobalLoadSAddrToVAddr.cpp
+ SIFixXcntStallSAddrReuse.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
SIInstrInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
similarity index 86%
rename from llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
rename to llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index d33396422264d..e9987d6b06bcc 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -1,4 +1,4 @@
-//===- SIGlobalLoadSAddrToVAddr.cpp - SADDR global loads to VADDR ---------===//
+//===- SIFixXcntStallSAddrReuse.cpp - Fix xcnt stalls from SADDR reuse ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -10,10 +10,10 @@
/// address pair is redefined later in the block. This avoids s_wait_xcnt
/// stalls from XNACK replay of loads that share a reused SGPR pair.
///
-/// Runs after SGPR RA / SILowerSGPRSpills and before VGPR RA.
+/// Runs after SGPR RA and before VGPR RA.
//===----------------------------------------------------------------------===//
-#include "SIGlobalLoadSAddrToVAddr.h"
+#include "SIFixXcntStallSAddrReuse.h"
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -27,24 +27,25 @@
using namespace llvm;
-#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
+#define DEBUG_TYPE "si-fix-xcnt-stall-saddr-reuse"
STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
static cl::opt<unsigned> SAddrRedefWindow(
"amdgpu-saddr-redef-window",
- cl::desc("Max non-meta instructions to scan for saddr redef "
+ cl::desc("Maximum instructions to scan for saddr redefines "
"(0 = scan to end of block)"),
cl::init(0), cl::Hidden);
static cl::opt<unsigned> AddrLivenessGroupSize(
"amdgpu-saddr-vaddr-liveness-group",
- cl::desc("Max converted VADDR addresses kept simultaneously live"),
+ cl::desc("Maximum converted VADDRs kept simultaneously live, kill is "
+ "issued for each group"),
cl::init(16), cl::Hidden);
namespace {
-class SIGlobalLoadSAddrToVAddr {
+class SIFixXcntStallSAddrReuse {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
@@ -58,19 +59,19 @@ class SIGlobalLoadSAddrToVAddr {
bool run(MachineFunction &MF);
};
-class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
+class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
public:
static char ID;
- SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
+ SIFixXcntStallSAddrReuseLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- return SIGlobalLoadSAddrToVAddr().run(MF);
+ return SIFixXcntStallSAddrReuse().run(MF);
}
StringRef getPassName() const override {
- return "SI Global Load SAddr to VAddr";
+ return "SI Fix Xcnt Stall SAddr Reuse";
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -81,20 +82,20 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
} // end anonymous namespace
-char SIGlobalLoadSAddrToVAddrLegacy::ID = 0;
+char SIFixXcntStallSAddrReuseLegacy::ID = 0;
-INITIALIZE_PASS(SIGlobalLoadSAddrToVAddrLegacy, DEBUG_TYPE,
- "SI Global Load SAddr to VAddr", false, false)
+INITIALIZE_PASS(SIFixXcntStallSAddrReuseLegacy, DEBUG_TYPE,
+ "SI Fix Xcnt Stall SAddr Reuse", false, false)
-char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
- SIGlobalLoadSAddrToVAddrLegacy::ID;
+char &llvm::SIFixXcntStallSAddrReuseLegacyID =
+ SIFixXcntStallSAddrReuseLegacy::ID;
PreservedAnalyses
-SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
+SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- if (!SIGlobalLoadSAddrToVAddr().run(MF))
+ if (!SIFixXcntStallSAddrReuse().run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
@@ -113,7 +114,7 @@ static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
TRI.isSGPRReg(MRI, SAddr.getReg());
}
-bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
+bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
Register SAddr) {
unsigned NumScanned = 0;
MachineBasicBlock::iterator I = std::next(MI.getIterator());
@@ -131,7 +132,7 @@ bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
return false;
}
-bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
+bool SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
unsigned Opc = MI.getOpcode();
@@ -212,7 +213,7 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
return true;
}
-bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
+bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
SmallVector<Register, 8> GroupRegs;
MachineInstr *GroupLast = nullptr;
bool Changed = false;
@@ -252,7 +253,7 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
return Changed;
}
-bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
+bool SIFixXcntStallSAddrReuse::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
similarity index 60%
rename from llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
rename to llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
index b89815683dfc5..9419fef056b22 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
@@ -1,4 +1,4 @@
-//===- SIGlobalLoadSAddrToVAddr.h -------------------------------*- C++ -*-===//
+//===- SIFixXcntStallSAddrReuse.h -------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,15 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
-#define LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
+#define LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
#include "llvm/CodeGen/MachinePassManager.h"
namespace llvm {
-class SIGlobalLoadSAddrToVAddrPass
- : public PassInfoMixin<SIGlobalLoadSAddrToVAddrPass> {
+class SIFixXcntStallSAddrReusePass
+ : public PassInfoMixin<SIFixXcntStallSAddrReusePass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
@@ -22,4 +22,4 @@ class SIGlobalLoadSAddrToVAddrPass
} // namespace llvm
-#endif // LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#endif // LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 54b52520fe91e..a1e157cf50b7f 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,106 +1,61 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
---
-name: basic_convert
+name: global_sadd_load
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: basic_convert
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: global_sadd_load
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: basic_convert
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; GFX942-LABEL: name: global_sadd_load
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
S_ENDPGM 0, implicit %1
...
-# Default window=0 scans to end of block; redef after 5 NOPs still triggers.
---
-name: redef_after_nops
+name: global_sadd_store
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: redef_after_nops
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: global_sadd_store
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0
;
- ; DISABLED-LABEL: name: redef_after_nops
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- S_NOP 0
- S_NOP 0
- S_NOP 0
- S_NOP 0
- S_NOP 0
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
-...
-
----
-name: store_convert
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45
-
- ; CONVERT-LABEL: name: store_convert
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0
- ;
- ; DISABLED-LABEL: name: store_convert
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0
+ ; GFX942-LABEL: name: global_sadd_store
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
GLOBAL_STORE_DWORD_SADDR %0, %1, $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
@@ -115,26 +70,26 @@ body: |
bb.0:
liveins: $sgpr44_sgpr45, $vgpr10
- ; CONVERT-LABEL: name: nonzero_vaddr
- ; CONVERT: liveins: $sgpr44_sgpr45, $vgpr10
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
- ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
- ; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
- ; CONVERT-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
- ; CONVERT-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: nonzero_vaddr
+ ; GFX1250: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
+ ; GFX1250-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
+ ; GFX1250-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: nonzero_vaddr
- ; DISABLED: liveins: $sgpr44_sgpr45, $vgpr10
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; GFX942-LABEL: name: nonzero_vaddr
+ ; GFX942: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = COPY $vgpr10
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
@@ -148,30 +103,30 @@ body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: multi_load_kill
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
- ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: KILL killed [[COPY]], killed [[COPY1]]
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ ; GFX1250-LABEL: name: multi_load_kill
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: KILL killed [[COPY]], killed [[COPY1]]
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
- ; DISABLED-LABEL: name: multi_load_kill
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 1
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ ; GFX942-LABEL: name: multi_load_kill
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: $sgpr45 = S_MOV_B32 0
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 1
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
More information about the llvm-commits
mailing list