[llvm] [AMDGPU] Rewrite SADDR memory ops to VADDR reduce xcnt stalls (PR #190654)
Vigneshwar Jayakumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 13:02:45 PDT 2026
https://github.com/VigneshwarJ updated https://github.com/llvm/llvm-project/pull/190654
>From 4eb2c4b90fb9b7214493282b5a6a8736941e18bf Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Tue, 31 Mar 2026 13:26:45 -0500
Subject: [PATCH 01/16] [AMDGPU] Add SIGlobalLoadSAddrToVAddr pass to reduce
s_wait_xcnt stalls
On GFX1250, high SGPR pressure causes the register allocator to reuse
the same SGPR pair across consecutive global loads. Since the hardware
needs the SGPR address to remain valid for address translation replay,
SIInsertWaitcnts inserts s_wait_xcnt before every SGPR redefinition,
serializing the entire load sequence.
Add a pass that converts SADDR global loads/stores to VADDR form by
copying the SGPR address into a VGPR pair when the SGPR is about to be
overwritten. VGPR defs do not require s_wait_xcnt, eliminating the
stalls. The pass runs before VGPR allocation and inserts KILL pseudos
to prevent RA from reusing the same VGPR across converted loads.
Disabled by default; enable with -amdgpu-global-load-saddr-to-vaddr=1.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 10 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 274 ++++++++++++++++++
.../Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h | 25 ++
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 200 +++++++++++++
7 files changed, 514 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
create mode 100644 llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
create mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 878f374110159..16b3d423a45ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -214,6 +214,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
+void initializeSIGlobalLoadSAddrToVAddrLegacyPass(PassRegistry &);
+extern char &SIGlobalLoadSAddrToVAddrLegacyID;
+
void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
extern char &SILoadStoreOptimizerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 8a046e83548cc..84efd6452e008 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -133,6 +133,7 @@ MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
+MACHINE_FUNCTION_PASS("si-global-load-saddr-to-vaddr", SIGlobalLoadSAddrToVAddrPass())
MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index daa9f933fce59..6a986a16a37eb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -52,6 +52,7 @@
#include "SIFixVGPRCopies.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
+#include "SIGlobalLoadSAddrToVAddr.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -727,6 +728,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
+ initializeSIGlobalLoadSAddrToVAddrLegacyPass(*PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -1814,6 +1816,8 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
+ addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1851,6 +1855,8 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
+ addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -2455,6 +2461,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
+ addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2549,6 +2557,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
+ addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index cda288e012925..1b193a4b581fb 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -161,6 +161,7 @@ add_llvm_target(AMDGPUCodeGen
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
SIFrameLowering.cpp
+ SIGlobalLoadSAddrToVAddr.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
SIInstrInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
new file mode 100644
index 0000000000000..838b22132943f
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -0,0 +1,274 @@
+//===-- SIGlobalLoadSAddrToVAddr.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Convert SADDR global loads/stores to VADDR form when the SGPR address pair
+// is about to be overwritten. This eliminates s_wait_xcnt hazards that
+// serialize loads sharing a single SGPR scratch address register.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SIGlobalLoadSAddrToVAddr.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
+
+static cl::opt<bool> EnableSAddrToVAddr(
+ "amdgpu-global-load-saddr-to-vaddr",
+ cl::desc(
+ "Convert SADDR global loads/stores to VADDR when saddr is overwritten"),
+ cl::Hidden);
+
+static cl::opt<unsigned> SAddrToVAddrWindow(
+ "amdgpu-global-load-saddr-to-vaddr-window",
+ cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
+ cl::Hidden);
+
+static bool isEnabled() {
+ if (EnableSAddrToVAddr.getNumOccurrences())
+ return EnableSAddrToVAddr;
+ if (const char *Env = std::getenv("AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR"))
+ return StringRef(Env) != "0";
+ return false;
+}
+
+namespace {
+
+class SIGlobalLoadSAddrToVAddr {
+ const SIInstrInfo *TII = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+ MachineRegisterInfo *MRI = nullptr;
+
+ bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
+ Register &NewAddrReg);
+ bool sAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+
+public:
+ bool run(MachineFunction &MF);
+};
+
+class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (!isEnabled())
+ return false;
+ SIGlobalLoadSAddrToVAddr Impl;
+ return Impl.run(MF);
+ }
+
+ StringRef getPassName() const override {
+ return "SI Global Load SAddr to VAddr";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // end anonymous namespace
+
+char SIGlobalLoadSAddrToVAddrLegacy::ID = 0;
+
+INITIALIZE_PASS(SIGlobalLoadSAddrToVAddrLegacy, DEBUG_TYPE,
+ "SI Global Load SAddr to VAddr", false, false)
+
+char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
+ SIGlobalLoadSAddrToVAddrLegacy::ID;
+
+PreservedAnalyses
+SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!isEnabled())
+ return PreservedAnalyses::all();
+ SIGlobalLoadSAddrToVAddr Impl;
+ if (!Impl.run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedInWindow(MachineInstr &MI,
+ Register SAddr) {
+ unsigned Count = 0;
+ MachineBasicBlock::iterator Next = std::next(MI.getIterator());
+ MachineBasicBlock::iterator End = MI.getParent()->end();
+ for (; Next != End && Count < SAddrToVAddrWindow; ++Next) {
+ if (Next->isMetaInstruction())
+ continue;
+ ++Count;
+ for (const MachineOperand &MO : Next->operands()) {
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ Register Reg = MO.getReg();
+ if (Reg.isPhysical() && TRI->regsOverlap(Reg, SAddr))
+ return true;
+ }
+ }
+ return false;
+}
+
+bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
+ MachineBasicBlock &MBB,
+ Register &NewAddrReg) {
+ unsigned Opc = MI.getOpcode();
+ int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
+ if (NewOpc < 0)
+ return false;
+
+ int NewVAddrIdx =
+ AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
+ if (NewVAddrIdx < 0)
+ return false;
+
+ int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
+ assert(OldSAddrIdx >= 0 && OldVAddrIdx >= 0);
+
+ MachineOperand &SAddr = MI.getOperand(OldSAddrIdx);
+ MachineOperand &VAddr = MI.getOperand(OldVAddrIdx);
+ Register SAddrReg = SAddr.getReg();
+ Register OldVAddrReg = VAddr.getReg();
+
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ bool VAddrIsZero = false;
+ if (OldVAddrReg.isVirtual()) {
+ MachineInstr *VAddrDef = MRI->getVRegDef(OldVAddrReg);
+ if (VAddrDef &&
+ (VAddrDef->isImplicitDef() ||
+ (VAddrDef->isMoveImmediate() && VAddrDef->getOperand(1).isImm() &&
+ VAddrDef->getOperand(1).getImm() == 0)))
+ VAddrIsZero = true;
+ }
+
+ Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
+
+ if (VAddrIsZero) {
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr)
+ .addReg(SAddrReg);
+ } else {
+ MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
+ MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
+
+ Register VAddrHiExt =
+ MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
+ .addImm(31)
+ .addReg(OldVAddrReg);
+
+ Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo)
+ .addReg(SAddrLoPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi)
+ .addReg(SAddrHiPhys);
+
+ MCRegister VCC = TRI->getVCC();
+
+ auto AddLo =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), NewVAddr)
+ .addDef(VCC)
+ .addReg(SAddrLo)
+ .addReg(OldVAddrReg)
+ .addImm(0);
+ AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
+ AddLo->getOperand(0).setIsUndef(true);
+
+ auto AddHi =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
+ .addDef(VCC, RegState::Dead)
+ .addReg(SAddrHi)
+ .addReg(VAddrHiExt)
+ .addReg(VCC, RegState::Kill)
+ .addImm(0);
+ AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
+ }
+
+ MI.setDesc(TII->get(NewOpc));
+ VAddr.setReg(NewVAddr);
+ MI.removeOperand(OldSAddrIdx);
+
+ LLVM_DEBUG(dbgs() << " Converted to VADDR: " << MI);
+ NewAddrReg = NewVAddr;
+ return true;
+}
+
+bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ TII = ST.getInstrInfo();
+ TRI = ST.getRegisterInfo();
+ MRI = &MF.getRegInfo();
+
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : MF) {
+ SmallVector<Register, 32> AddrRegs;
+ MachineInstr *LastConverted = nullptr;
+
+ for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
+ MachineInstr &MI = *MII;
+ unsigned Opc = MI.getOpcode();
+
+ int SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0)
+ continue;
+
+ if (!TII->isFLATGlobal(MI))
+ continue;
+
+ MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ if (!SAddr.isReg() || !SAddr.getReg().isPhysical())
+ continue;
+
+ if (!TRI->isSGPRReg(*MRI, SAddr.getReg()))
+ continue;
+
+ if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
+ continue;
+
+ LLVM_DEBUG(dbgs() << "SAddr redef in window: " << MI);
+ Register NewAddr;
+ if (convertToVAddr(MI, MBB, NewAddr)) {
+ AddrRegs.push_back(NewAddr);
+ LastConverted = &MI;
+ Changed = true;
+ }
+ }
+
+ // Insert KILL after the last converted load to extend the live ranges of
+ // all address VGPRs. This prevents the register allocator from reusing
+ // the same physical registers, which would introduce WAR hazards and
+ // require s_wait_xcnt between consecutive loads.
+ if (LastConverted && AddrRegs.size() > 1) {
+ auto InsertPt = std::next(LastConverted->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, LastConverted->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : AddrRegs)
+ KillMI.addReg(R);
+ }
+ }
+
+ return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
new file mode 100644
index 0000000000000..b89815683dfc5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
@@ -0,0 +1,25 @@
+//===- SIGlobalLoadSAddrToVAddr.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#define LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class SIGlobalLoadSAddrToVAddrPass
+ : public PassInfoMixin<SIGlobalLoadSAddrToVAddrPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
new file mode 100644
index 0000000000000..574777309b1f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -0,0 +1,200 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=1 -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: env AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR=1 llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+
+# Test: Basic conversion - saddr redefined 1 instruction after load
+---
+name: basic_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: basic_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: basic_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: No conversion - saddr not redefined within window (5 nops > window=4)
+---
+name: no_redef_no_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: no_redef_no_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ;
+ ; DISABLED-LABEL: name: no_redef_no_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Window boundary - redef at exactly instruction 4 within window -> converted
+---
+name: window_boundary_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: window_boundary_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: S_NOP 0
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: window_boundary_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: S_NOP 0
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_NOP 0
+ S_NOP 0
+ S_NOP 0
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Store conversion
+---
+name: store_convert
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: store_convert
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0
+ ;
+ ; DISABLED-LABEL: name: store_convert
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD_SADDR %0, %1, $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0
+...
+
+# Test: Non-zero vaddr offset requires full add sequence
+---
+name: nonzero_vaddr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45, $vgpr10
+
+ ; CONVERT-LABEL: name: nonzero_vaddr
+ ; CONVERT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
+ ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
+ ; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
+ ; CONVERT-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
+ ; CONVERT-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ;
+ ; DISABLED-LABEL: name: nonzero_vaddr
+ ; DISABLED: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ %0:vgpr_32 = COPY $vgpr10
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ S_ENDPGM 0, implicit %1
+...
+
+# Test: Multiple converted loads get a KILL to extend address live ranges
+---
+name: multi_load_kill
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45
+
+ ; CONVERT-LABEL: name: multi_load_kill
+ ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
+ ; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
+ ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: KILL [[COPY]], [[COPY1]]
+ ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ ;
+ ; DISABLED-LABEL: name: multi_load_kill
+ ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
+ ; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
+ ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 1
+ ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 0
+ $sgpr45 = S_MOV_B32 0
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 1
+ S_ENDPGM 0, implicit %1, implicit %2
+...
>From cf335ac87d9966612e710151dc9bf45a0eed6aed Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 1 Apr 2026 08:26:06 -0500
Subject: [PATCH 02/16] isel change
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 11 +++++++
.../CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll | 29 +++++++++++++++++++
2 files changed, 40 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 598ab6f7354f5..a0d327e83166a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2103,6 +2103,17 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
isa<ConstantSDNode>(Addr))
return false;
+ // On targets with XCNT, avoid placing a pure-uniform address into SADDR
+ // with a zero voffset. When the RA reuses the SGPR pair across multiple
+ // loads (common under high SGPR pressure), SIInsertWaitcnts must serialize
+ // each load with s_wait_xcnt for XNACK replay safety. Falling through to
+ // the VADDR pattern lets addresses stay in VGPRs where the RA naturally
+ // assigns distinct registers, avoiding the hazard entirely.
+ // The add(sgpr, extend(vgpr)) patterns matched above still select SADDR
+ // since those have a stable SGPR base that is not recomputed per load.
+ if (Subtarget->hasWaitXcnt())
+ return false;
+
// It's cheaper to materialize a single 32-bit zero for vaddr than the two
// moves required to copy a 64-bit SGPR to VGPR.
SAddr = Addr;
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
new file mode 100644
index 0000000000000..3b46486332ca8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
@@ -0,0 +1,29 @@
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+
+; On GFX1250 (hasWaitXcnt), pure-uniform addresses should use VADDR to
+; avoid s_wait_xcnt serialization when the RA reuses the SGPR pair.
+; On GFX12, SADDR is still preferred for pure-uniform addresses.
+
+define amdgpu_ps float @pure_uniform_global_load(ptr addrspace(1) inreg %sbase) {
+; GFX1250-LABEL: pure_uniform_global_load:
+; GFX1250: global_load_b32 v0, v[0:1], off
+;
+; GFX12-LABEL: pure_uniform_global_load:
+; GFX12: global_load_b32 v0, v0, s[2:3]
+ %val = load volatile float, ptr addrspace(1) %sbase
+ ret float %val
+}
+
+; Verify that add(sgpr, zext(vgpr)) still selects SADDR on GFX1250.
+define amdgpu_ps float @sgpr_plus_vgpr_offset(ptr addrspace(1) inreg %sbase, i32 %voffset) {
+; GFX1250-LABEL: sgpr_plus_vgpr_offset:
+; GFX1250: global_load_b32 v0, v0, s[2:3]
+;
+; GFX12-LABEL: sgpr_plus_vgpr_offset:
+; GFX12: global_load_b32 v0, v0, s[2:3]
+ %zext = zext i32 %voffset to i64
+ %gep = getelementptr i8, ptr addrspace(1) %sbase, i64 %zext
+ %val = load volatile float, ptr addrspace(1) %gep
+ ret float %val
+}
>From ea277712dd78c766b60f4e835307120bfe48d5a6 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Fri, 3 Apr 2026 13:18:55 -0500
Subject: [PATCH 03/16] Revert "isel change"
This reverts commit cf335ac87d9966612e710151dc9bf45a0eed6aed.
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 11 -------
.../CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll | 29 -------------------
2 files changed, 40 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a0d327e83166a..598ab6f7354f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2103,17 +2103,6 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
isa<ConstantSDNode>(Addr))
return false;
- // On targets with XCNT, avoid placing a pure-uniform address into SADDR
- // with a zero voffset. When the RA reuses the SGPR pair across multiple
- // loads (common under high SGPR pressure), SIInsertWaitcnts must serialize
- // each load with s_wait_xcnt for XNACK replay safety. Falling through to
- // the VADDR pattern lets addresses stay in VGPRs where the RA naturally
- // assigns distinct registers, avoiding the hazard entirely.
- // The add(sgpr, extend(vgpr)) patterns matched above still select SADDR
- // since those have a stable SGPR base that is not recomputed per load.
- if (Subtarget->hasWaitXcnt())
- return false;
-
// It's cheaper to materialize a single 32-bit zero for vaddr than the two
// moves required to copy a 64-bit SGPR to VGPR.
SAddr = Addr;
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
deleted file mode 100644
index 3b46486332ca8..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-xcnt-avoid.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-
-; On GFX1250 (hasWaitXcnt), pure-uniform addresses should use VADDR to
-; avoid s_wait_xcnt serialization when the RA reuses the SGPR pair.
-; On GFX12, SADDR is still preferred for pure-uniform addresses.
-
-define amdgpu_ps float @pure_uniform_global_load(ptr addrspace(1) inreg %sbase) {
-; GFX1250-LABEL: pure_uniform_global_load:
-; GFX1250: global_load_b32 v0, v[0:1], off
-;
-; GFX12-LABEL: pure_uniform_global_load:
-; GFX12: global_load_b32 v0, v0, s[2:3]
- %val = load volatile float, ptr addrspace(1) %sbase
- ret float %val
-}
-
-; Verify that add(sgpr, zext(vgpr)) still selects SADDR on GFX1250.
-define amdgpu_ps float @sgpr_plus_vgpr_offset(ptr addrspace(1) inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: sgpr_plus_vgpr_offset:
-; GFX1250: global_load_b32 v0, v0, s[2:3]
-;
-; GFX12-LABEL: sgpr_plus_vgpr_offset:
-; GFX12: global_load_b32 v0, v0, s[2:3]
- %zext = zext i32 %voffset to i64
- %gep = getelementptr i8, ptr addrspace(1) %sbase, i64 %zext
- %val = load volatile float, ptr addrspace(1) %gep
- ret float %val
-}
>From 91b62dbd8b9d2ff44e4201ddf21bf3bc569d28c5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Fri, 3 Apr 2026 13:44:40 -0500
Subject: [PATCH 04/16] new_changes
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 78 +++++++++++-----
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 6 +-
.../wait-xcnt-atomic-rmw-optimization.ll | 89 ++++++++++---------
.../waitcnt-loop-ds-prefetch-pattern.ll | 29 +++---
4 files changed, 121 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 838b22132943f..0d3090c478b5c 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -17,6 +17,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -40,12 +41,15 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
cl::Hidden);
-static bool isEnabled() {
+static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
+ "amdgpu-global-load-saddr-to-vaddr-kill-group-size",
+ cl::desc("Max converted loads per KILL group (0 = single KILL per block)"),
+ cl::init(16), cl::Hidden);
+
+static bool isEnabled(const GCNSubtarget &ST) {
if (EnableSAddrToVAddr.getNumOccurrences())
return EnableSAddrToVAddr;
- if (const char *Env = std::getenv("AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR"))
- return StringRef(Env) != "0";
- return false;
+ return ST.hasWaitXcnt();
}
namespace {
@@ -54,6 +58,7 @@ class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
@@ -70,7 +75,8 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- if (!isEnabled())
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!isEnabled(ST))
return false;
SIGlobalLoadSAddrToVAddr Impl;
return Impl.run(MF);
@@ -99,7 +105,8 @@ char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
PreservedAnalyses
SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- if (!isEnabled())
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!isEnabled(ST))
return PreservedAnalyses::all();
SIGlobalLoadSAddrToVAddr Impl;
if (!Impl.run(MF))
@@ -221,11 +228,37 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
+ const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+ unsigned Occupancy = MFI->getOccupancy();
+ unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+
+ // Reserve at most a quarter of the VGPR budget for converted address
+ // registers. Each conversion creates one vreg_64 kept live to the
+ // group's KILL, so the cost is 2 VGPRs per conversion.
+ MaxAddrVGPRsPerBB = MaxVGPRs / 4;
+
+ LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
+ << " maxVGPRs=" << MaxVGPRs
+ << " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
+
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
- SmallVector<Register, 32> AddrRegs;
- MachineInstr *LastConverted = nullptr;
+ SmallVector<Register, 8> GroupRegs;
+ MachineInstr *GroupLast = nullptr;
+ unsigned TotalAddrVGPRs = 0;
+
+ auto FlushGroup = [&]() {
+ if (GroupLast && GroupRegs.size() > 1) {
+ auto InsertPt = std::next(GroupLast->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : GroupRegs)
+ KillMI.addReg(R);
+ }
+ GroupRegs.clear();
+ GroupLast = nullptr;
+ };
for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
MachineInstr &MI = *MII;
@@ -248,26 +281,27 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
continue;
- LLVM_DEBUG(dbgs() << "SAddr redef in window: " << MI);
+ // Stop converting in this BB if we would exceed the VGPR budget.
+ if (TotalAddrVGPRs + 2 > MaxAddrVGPRsPerBB) {
+ LLVM_DEBUG(dbgs() << " Skipping (VGPR budget exhausted): " << MI);
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
Register NewAddr;
if (convertToVAddr(MI, MBB, NewAddr)) {
- AddrRegs.push_back(NewAddr);
- LastConverted = &MI;
+ GroupRegs.push_back(NewAddr);
+ GroupLast = &MI;
+ TotalAddrVGPRs += 2;
Changed = true;
+
+ if (SAddrToVAddrKillGroupSize > 0 &&
+ GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ FlushGroup();
}
}
- // Insert KILL after the last converted load to extend the live ranges of
- // all address VGPRs. This prevents the register allocator from reusing
- // the same physical registers, which would introduce WAR hazards and
- // require s_wait_xcnt between consecutive loads.
- if (LastConverted && AddrRegs.size() > 1) {
- auto InsertPt = std::next(LastConverted->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, LastConverted->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
- for (Register R : AddrRegs)
- KillMI.addReg(R);
- }
+ FlushGroup();
}
return Changed;
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 574777309b1f0..2d94c4c1c67f3 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=1 -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
-# RUN: env AMDGPU_GLOBAL_LOAD_SADDR_TO_VADDR=1 llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=0 -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
# Test: Basic conversion - saddr redefined 1 instruction after load
---
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 0f6edafb22908..0c29a1ee71071 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -965,57 +965,60 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
+; GFX1250-NEXT: v_writelane_b32 v5, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s3, 1
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s1, 2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v5, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: .LBB17_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 3
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 2
-; GFX1250-NEXT: v_readlane_b32 s2, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s3, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s2, 1
; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB17_1
@@ -1047,48 +1050,51 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
-; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v5, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: v_mov_b32_e32 v2, 1
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: v_mov_b32_e32 v4, 2
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s0, -1
; GFX1250-NEXT: s_mov_b32 s2, 0
; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
; GFX1250-NEXT: .LBB18_1: ; %Flow
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 2
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 2
; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX1250-NEXT: s_mov_b32 s0, 1
; GFX1250-NEXT: v_cmp_ne_u32_e64 s0, v0, s0
@@ -1096,12 +1102,12 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
; GFX1250-NEXT: ; %bb.2: ; %bb1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 3
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1117,39 +1123,40 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_branch .LBB18_4
; GFX1250-NEXT: .LBB18_3: ; %bb2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: v_mov_b32_e32 v2, 4
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_branch .LBB18_1
; GFX1250-NEXT: .LBB18_4: ; %merge
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 5
; GFX1250-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index 7f98f8cf8de06..af53cb587bc7d 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -12,14 +12,14 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
; CHECK-NEXT: v_and_b32_e32 v12, 0x3ff, v0
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v6, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, v4
+; CHECK-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8
+; CHECK-NEXT: v_mov_b32_e32 v11, v8
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s1
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: ds_load_b128 v[8:11], v13
+; CHECK-NEXT: ds_load_b128 v[4:7], v13
; CHECK-NEXT: ds_load_b128 v[0:3], v13 offset:16
; CHECK-NEXT: s_wait_dscnt 0x0
; CHECK-NEXT: .LBB0_1: ; %loop
@@ -27,24 +27,23 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_barrier_signal -1
; CHECK-NEXT: s_add_co_i32 s1, s1, 1
; CHECK-NEXT: s_wait_dscnt 0x1
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[10:11]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[8:9]
-; CHECK-NEXT: v_lshl_add_u32 v14, s1, 5, v13
+; CHECK-NEXT: v_pk_add_f32 v[6:7], v[10:11], v[6:7]
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[8:9], v[4:5]
+; CHECK-NEXT: v_lshl_add_u32 v10, s1, 5, v13
; CHECK-NEXT: s_cmp_lt_i32 s1, s0
; CHECK-NEXT: s_wait_dscnt 0x0
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[2:3]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[0:1]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[6:7]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[4:5]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], v[6:7], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[14:15], v[4:5], v[0:1]
; CHECK-NEXT: s_barrier_wait -1
-; CHECK-NEXT: ds_load_b128 v[8:11], v14
-; CHECK-NEXT: ds_load_b128 v[0:3], v14 offset:16
+; CHECK-NEXT: ds_load_b128 v[4:7], v10
+; CHECK-NEXT: ds_load_b128 v[0:3], v10 offset:16
+; CHECK-NEXT: v_pk_add_f32 v[10:11], v[8:9], v[8:9]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], v[14:15], v[14:15]
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: global_store_b128 v12, v[4:7], s[0:1] scale_offset
+; CHECK-NEXT: global_store_b128 v12, v[8:11], s[0:1] scale_offset
; CHECK-NEXT: s_endpgm
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
>From c8536d667046dd894e8dccb6452b5a135760a3b5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Sat, 4 Apr 2026 10:10:58 -0500
Subject: [PATCH 05/16] saddr
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 286 +++++++++++-------
.../CodeGen/AMDGPU/saddr-to-vaddr-budget.mir | 52 ++++
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 82 +++--
3 files changed, 260 insertions(+), 160 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 0d3090c478b5c..802425daf7bc8 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -1,4 +1,4 @@
-//===-- SIGlobalLoadSAddrToVAddr.cpp ---------------------------------------===//
+//===- SIGlobalLoadSAddrToVAddr.cpp - SADDR global loads to VADDR ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,9 +6,9 @@
//
//===----------------------------------------------------------------------===//
//
-// Convert SADDR global loads/stores to VADDR form when the SGPR address pair
-// is about to be overwritten. This eliminates s_wait_xcnt hazards that
-// serialize loads sharing a single SGPR scratch address register.
+/// \file Convertglobal SADDR memory ops to VADDR when the SGPR address is
+/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
+/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills;
//
//===----------------------------------------------------------------------===//
@@ -19,8 +19,11 @@
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -30,12 +33,6 @@ using namespace llvm;
#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
-static cl::opt<bool> EnableSAddrToVAddr(
- "amdgpu-global-load-saddr-to-vaddr",
- cl::desc(
- "Convert SADDR global loads/stores to VADDR when saddr is overwritten"),
- cl::Hidden);
-
static cl::opt<unsigned> SAddrToVAddrWindow(
"amdgpu-global-load-saddr-to-vaddr-window",
cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
@@ -43,26 +40,39 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
"amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group (0 = single KILL per block)"),
+ cl::desc("Max converted loads per KILL group (0 disables KILL insertion)"),
cl::init(16), cl::Hidden);
-static bool isEnabled(const GCNSubtarget &ST) {
- if (EnableSAddrToVAddr.getNumOccurrences())
- return EnableSAddrToVAddr;
- return ST.hasWaitXcnt();
-}
+static cl::opt<unsigned> SAddrToVAddrMaxConverts(
+ "amdgpu-global-load-saddr-to-vaddr-max-converts",
+ cl::desc(
+ "Override max conversions per BB (0 = use occupancy-based budget)"),
+ cl::init(0), cl::Hidden);
namespace {
+struct Candidate {
+ MachineInstr *MI;
+ unsigned Pressure;
+};
+
class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ MachineFunction *CurMF = nullptr;
unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
- bool sAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+ bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
+
+ unsigned countLiveSGPR32(const LiveRegUnits &LRU) const;
+
+ void computeSGPRPressure(MachineBasicBlock &MBB,
+ DenseMap<MachineInstr *, unsigned> &Out);
+
+ bool processMBB(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF);
@@ -75,8 +85,7 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!isEnabled(ST))
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
SIGlobalLoadSAddrToVAddr Impl;
return Impl.run(MF);
@@ -105,8 +114,7 @@ char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
PreservedAnalyses
SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- if (!isEnabled(ST))
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
SIGlobalLoadSAddrToVAddr Impl;
if (!Impl.run(MF))
@@ -116,26 +124,66 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
return PA;
}
-bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedInWindow(MachineInstr &MI,
- Register SAddr) {
+unsigned
+SIGlobalLoadSAddrToVAddr::countLiveSGPR32(const LiveRegUnits &LRU) const {
unsigned Count = 0;
- MachineBasicBlock::iterator Next = std::next(MI.getIterator());
- MachineBasicBlock::iterator End = MI.getParent()->end();
- for (; Next != End && Count < SAddrToVAddrWindow; ++Next) {
- if (Next->isMetaInstruction())
+ for (MCPhysReg Reg : TRI->getAllSGPR32(*CurMF)) {
+ if (!LRU.available(Reg))
+ ++Count;
+ }
+ return Count;
+}
+
+void SIGlobalLoadSAddrToVAddr::computeSGPRPressure(
+ MachineBasicBlock &MBB, DenseMap<MachineInstr *, unsigned> &Out) {
+ Out.clear();
+ LiveRegUnits Live(*TRI);
+ Live.addLiveOuts(MBB);
+
+ for (MachineInstr &MI : reverse(MBB)) {
+ Live.stepBackward(MI);
+ Out[&MI] = countLiveSGPR32(Live);
+ }
+}
+
+bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
+ Register SAddr) {
+ unsigned NumScanned = 0;
+ auto I = std::next(MI.getIterator());
+ auto E = MI.getParent()->end();
+ for (; I != E; ++I) {
+ if (I->isMetaInstruction())
continue;
- ++Count;
- for (const MachineOperand &MO : Next->operands()) {
- if (!MO.isReg() || !MO.isDef())
- continue;
- Register Reg = MO.getReg();
- if (Reg.isPhysical() && TRI->regsOverlap(Reg, SAddr))
+
+ ++NumScanned;
+
+ for (const MachineOperand &MO : I->operands()) {
+ if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
+ TRI->regsOverlap(MO.getReg(), SAddr))
return true;
}
+
+ if (NumScanned >= SAddrToVAddrWindow)
+ return false;
}
return false;
}
+/// Return true if \p MI is a FLAT global memory op with a physical SGPR saddr.
+static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
+ const SIInstrInfo &TII,
+ const SIRegisterInfo &TRI,
+ const MachineRegisterInfo &MRI,
+ int &SAddrIdx) {
+ unsigned Opc = MI.getOpcode();
+ SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ return false;
+ const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ return SAddr.isReg() && SAddr.getReg().isPhysical() &&
+ TRI.isSGPRReg(MRI, SAddr.getReg());
+}
+
bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
@@ -144,8 +192,7 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
if (NewOpc < 0)
return false;
- int NewVAddrIdx =
- AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
+ int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
if (NewVAddrIdx < 0)
return false;
@@ -157,7 +204,6 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineOperand &VAddr = MI.getOperand(OldVAddrIdx);
Register SAddrReg = SAddr.getReg();
Register OldVAddrReg = VAddr.getReg();
-
const DebugLoc &DL = MI.getDebugLoc();
bool VAddrIsZero = false;
@@ -173,24 +219,24 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
if (VAddrIsZero) {
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr)
- .addReg(SAddrReg);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
} else {
+ assert(OldVAddrReg.isVirtual() &&
+ TRI->getRegSizeInBits(*MRI->getRegClass(OldVAddrReg)) == 32 &&
+ "Non-zero vaddr path expects a 32-bit VGPR operand");
+
MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
- Register VAddrHiExt =
- MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register VAddrHiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
.addImm(31)
.addReg(OldVAddrReg);
Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo)
- .addReg(SAddrLoPhys);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi)
- .addReg(SAddrHiPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo).addReg(SAddrLoPhys);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi).addReg(SAddrHiPhys);
MCRegister VCC = TRI->getVCC();
@@ -222,87 +268,101 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
return true;
}
+bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
+ SmallVector<MachineInstr *, 16> EligibleMIs;
+ for (MachineInstr &MI : MBB) {
+ int SAddrIdx;
+ if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ continue;
+ if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
+ continue;
+ EligibleMIs.push_back(&MI);
+ }
+
+ if (EligibleMIs.empty())
+ return false;
+
+ DenseMap<MachineInstr *, unsigned> PressureMap;
+ computeSGPRPressure(MBB, PressureMap);
+
+ SmallVector<Candidate, 16> Candidates;
+ for (MachineInstr *MI : EligibleMIs)
+ Candidates.push_back({MI, PressureMap.lookup(MI)});
+
+ // Prefer highest SGPR pressure; each conversion costs one vreg_64.
+ const unsigned MaxConverts = SAddrToVAddrMaxConverts > 0
+ ? SAddrToVAddrMaxConverts.getValue()
+ : MaxAddrVGPRsPerBB / 2;
+ DenseSet<MachineInstr *> Allowed;
+ llvm::sort(Candidates, [](const Candidate &A, const Candidate &B) {
+ return A.Pressure > B.Pressure;
+ });
+ const unsigned NumToConvert =
+ std::min(static_cast<unsigned>(Candidates.size()), MaxConverts);
+ for (unsigned I = 0; I < NumToConvert; ++I)
+ Allowed.insert(Candidates[I].MI);
+
+ SmallVector<Register, 8> GroupRegs;
+ MachineInstr *GroupLast = nullptr;
+ bool Changed = false;
+
+ // KILL extends address VGPR liveness to prevent RA reuse within a group.
+ auto FlushGroup = [&]() {
+ if (GroupLast && GroupRegs.size() > 1) {
+ auto InsertPt = std::next(GroupLast->getIterator());
+ auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
+ TII->get(TargetOpcode::KILL));
+ for (Register R : GroupRegs)
+ KillMI.addReg(R, RegState::Kill);
+ }
+ GroupRegs.clear();
+ GroupLast = nullptr;
+ };
+
+ for (MachineInstr &MI : MBB) {
+ if (!Allowed.count(&MI))
+ continue;
+
+ LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
+ Register NewAddr;
+ if (convertToVAddr(MI, MBB, NewAddr)) {
+ GroupRegs.push_back(NewAddr);
+ GroupLast = &MI;
+ Changed = true;
+
+ if (SAddrToVAddrKillGroupSize > 0 &&
+ GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ FlushGroup();
+ }
+ }
+
+ FlushGroup();
+ return Changed;
+}
+
bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
+ CurMF = &MF;
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
unsigned Occupancy = MFI->getOccupancy();
- unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+ unsigned DynBlockSize = MFI->getDynamicVGPRBlockSize();
+ unsigned VGPRsAtOccupancy = ST.getMaxNumVGPRs(Occupancy, DynBlockSize);
- // Reserve at most a quarter of the VGPR budget for converted address
- // registers. Each conversion creates one vreg_64 kept live to the
- // group's KILL, so the cost is 2 VGPRs per conversion.
- MaxAddrVGPRsPerBB = MaxVGPRs / 4;
+ // Cap address VGPRs per BB to half the per-wave budget at target occupancy.
+ MaxAddrVGPRsPerBB = VGPRsAtOccupancy / 2;
LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
- << " maxVGPRs=" << MaxVGPRs
+ << " VGPRsAtOcc=" << VGPRsAtOccupancy
<< " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
bool Changed = false;
-
- for (MachineBasicBlock &MBB : MF) {
- SmallVector<Register, 8> GroupRegs;
- MachineInstr *GroupLast = nullptr;
- unsigned TotalAddrVGPRs = 0;
-
- auto FlushGroup = [&]() {
- if (GroupLast && GroupRegs.size() > 1) {
- auto InsertPt = std::next(GroupLast->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
- for (Register R : GroupRegs)
- KillMI.addReg(R);
- }
- GroupRegs.clear();
- GroupLast = nullptr;
- };
-
- for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE; ++MII) {
- MachineInstr &MI = *MII;
- unsigned Opc = MI.getOpcode();
-
- int SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
- if (SAddrIdx < 0)
- continue;
-
- if (!TII->isFLATGlobal(MI))
- continue;
-
- MachineOperand &SAddr = MI.getOperand(SAddrIdx);
- if (!SAddr.isReg() || !SAddr.getReg().isPhysical())
- continue;
-
- if (!TRI->isSGPRReg(*MRI, SAddr.getReg()))
- continue;
-
- if (!sAddrRedefinedInWindow(MI, SAddr.getReg()))
- continue;
-
- // Stop converting in this BB if we would exceed the VGPR budget.
- if (TotalAddrVGPRs + 2 > MaxAddrVGPRsPerBB) {
- LLVM_DEBUG(dbgs() << " Skipping (VGPR budget exhausted): " << MI);
- continue;
- }
-
- LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
- Register NewAddr;
- if (convertToVAddr(MI, MBB, NewAddr)) {
- GroupRegs.push_back(NewAddr);
- GroupLast = &MI;
- TotalAddrVGPRs += 2;
- Changed = true;
-
- if (SAddrToVAddrKillGroupSize > 0 &&
- GroupRegs.size() >= SAddrToVAddrKillGroupSize)
- FlushGroup();
- }
- }
-
- FlushGroup();
- }
+ for (MachineBasicBlock &MBB : MF)
+ Changed |= processMBB(MBB);
return Changed;
}
+
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
new file mode 100644
index 0000000000000..8f5c706d7504e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
+# RUN: -amdgpu-global-load-saddr-to-vaddr-max-converts=1 -o - %s \
+# RUN: | FileCheck -check-prefix=CAP1 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
+# RUN: -o - %s | FileCheck -check-prefix=NOBUDGET %s
+
+# Test: With max-converts=1, only the highest SGPR-pressure candidate is
+# converted. Candidate 1 has SGPR pressure 4 (sgpr44, sgpr45, sgpr46, sgpr47
+# all live), candidate 2 has pressure 2 (only sgpr44, sgpr45 live).
+# The pass picks candidate 1 for conversion and leaves candidate 2 as SADDR.
+# Without the budget cap, both are converted.
+---
+name: budget_cap_picks_highest_pressure
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+
+ ; CAP1-LABEL: name: budget_cap_picks_highest_pressure
+ ; CAP1: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+ ; CAP1-NEXT: {{ $}}
+ ; CAP1-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CAP1-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CAP1-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
+ ; CAP1-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
+ ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CAP1-NEXT: $sgpr44 = S_MOV_B32 1
+ ; CAP1-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ;
+ ; NOBUDGET-LABEL: name: budget_cap_picks_highest_pressure
+ ; NOBUDGET: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
+ ; NOBUDGET-NEXT: {{ $}}
+ ; NOBUDGET-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; NOBUDGET-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
+ ; NOBUDGET-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
+ ; NOBUDGET-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; NOBUDGET-NEXT: KILL killed [[COPY]], killed [[COPY1]]
+ ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 1
+ ; NOBUDGET-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 $sgpr46
+ $sgpr45 = S_MOV_B32 $sgpr47
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ $sgpr44 = S_MOV_B32 1
+ S_ENDPGM 0, implicit %1, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 2d94c4c1c67f3..f05d7968c3838 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,5 +1,5 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -amdgpu-global-load-saddr-to-vaddr=0 -o - %s | FileCheck -check-prefix=DISABLED %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
# Test: Basic conversion - saddr redefined 1 instruction after load
@@ -11,14 +11,18 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: basic_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
; DISABLED-LABEL: name: basic_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
@@ -37,7 +41,9 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: no_redef_no_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
@@ -48,7 +54,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
;
; DISABLED-LABEL: name: no_redef_no_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: S_NOP 0
; DISABLED-NEXT: S_NOP 0
@@ -68,41 +76,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Window boundary - redef at exactly instruction 4 within window -> converted
----
-name: window_boundary_convert
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45
-
- ; CONVERT-LABEL: name: window_boundary_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
- ;
- ; DISABLED-LABEL: name: window_boundary_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- S_NOP 0
- S_NOP 0
- S_NOP 0
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
-...
-
# Test: Store conversion
---
name: store_convert
@@ -112,7 +85,9 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: store_convert
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
@@ -120,7 +95,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0
;
; DISABLED-LABEL: name: store_convert
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
@@ -141,7 +118,9 @@ body: |
liveins: $sgpr44_sgpr45, $vgpr10
; CONVERT-LABEL: name: nonzero_vaddr
- ; CONVERT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; CONVERT: liveins: $sgpr44_sgpr45, $vgpr10
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
@@ -152,7 +131,9 @@ body: |
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
; DISABLED-LABEL: name: nonzero_vaddr
- ; DISABLED: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DISABLED: liveins: $sgpr44_sgpr45, $vgpr10
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
@@ -171,19 +152,23 @@ body: |
liveins: $sgpr44_sgpr45
; CONVERT-LABEL: name: multi_load_kill
- ; CONVERT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CONVERT: liveins: $sgpr44_sgpr45
+ ; CONVERT-NEXT: {{ $}}
+ ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: KILL [[COPY]], [[COPY1]]
+ ; CONVERT-NEXT: KILL killed [[COPY]], killed [[COPY1]]
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
; DISABLED-LABEL: name: multi_load_kill
- ; DISABLED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; DISABLED: liveins: $sgpr44_sgpr45
+ ; DISABLED-NEXT: {{ $}}
+ ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
@@ -198,3 +183,6 @@ body: |
$sgpr44 = S_MOV_B32 1
S_ENDPGM 0, implicit %1, implicit %2
...
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# CONVERT: {{.*}}
+# DISABLED: {{.*}}
>From 58ff4cbe4af29f3490db35b345afe46801d8ae63 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Sat, 4 Apr 2026 10:36:24 -0500
Subject: [PATCH 06/16] removed budget
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 107 ++----------------
.../CodeGen/AMDGPU/saddr-to-vaddr-budget.mir | 52 ---------
2 files changed, 7 insertions(+), 152 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 802425daf7bc8..64212ede8b3c2 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -6,9 +6,9 @@
//
//===----------------------------------------------------------------------===//
//
-/// \file Convertglobal SADDR memory ops to VADDR when the SGPR address is
+/// \file Convert global SADDR memory ops to VADDR when the SGPR address is
/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
-/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills;
+/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills.
//
//===----------------------------------------------------------------------===//
@@ -17,13 +17,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
-#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -40,38 +34,18 @@ static cl::opt<unsigned> SAddrToVAddrWindow(
static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
"amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group (0 disables KILL insertion)"),
- cl::init(16), cl::Hidden);
-
-static cl::opt<unsigned> SAddrToVAddrMaxConverts(
- "amdgpu-global-load-saddr-to-vaddr-max-converts",
- cl::desc(
- "Override max conversions per BB (0 = use occupancy-based budget)"),
- cl::init(0), cl::Hidden);
+ cl::desc("Max converted loads per KILL group"), cl::init(16), cl::Hidden);
namespace {
-struct Candidate {
- MachineInstr *MI;
- unsigned Pressure;
-};
-
class SIGlobalLoadSAddrToVAddr {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
- MachineFunction *CurMF = nullptr;
- unsigned MaxAddrVGPRsPerBB = 0;
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
-
- unsigned countLiveSGPR32(const LiveRegUnits &LRU) const;
-
- void computeSGPRPressure(MachineBasicBlock &MBB,
- DenseMap<MachineInstr *, unsigned> &Out);
-
bool processMBB(MachineBasicBlock &MBB);
public:
@@ -124,28 +98,6 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
return PA;
}
-unsigned
-SIGlobalLoadSAddrToVAddr::countLiveSGPR32(const LiveRegUnits &LRU) const {
- unsigned Count = 0;
- for (MCPhysReg Reg : TRI->getAllSGPR32(*CurMF)) {
- if (!LRU.available(Reg))
- ++Count;
- }
- return Count;
-}
-
-void SIGlobalLoadSAddrToVAddr::computeSGPRPressure(
- MachineBasicBlock &MBB, DenseMap<MachineInstr *, unsigned> &Out) {
- Out.clear();
- LiveRegUnits Live(*TRI);
- Live.addLiveOuts(MBB);
-
- for (MachineInstr &MI : reverse(MBB)) {
- Live.stepBackward(MI);
- Out[&MI] = countLiveSGPR32(Live);
- }
-}
-
bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
Register SAddr) {
unsigned NumScanned = 0;
@@ -169,7 +121,6 @@ bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
return false;
}
-/// Return true if \p MI is a FLAT global memory op with a physical SGPR saddr.
static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
const SIInstrInfo &TII,
const SIRegisterInfo &TRI,
@@ -269,39 +220,6 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
}
bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
- SmallVector<MachineInstr *, 16> EligibleMIs;
- for (MachineInstr &MI : MBB) {
- int SAddrIdx;
- if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
- continue;
- if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
- continue;
- EligibleMIs.push_back(&MI);
- }
-
- if (EligibleMIs.empty())
- return false;
-
- DenseMap<MachineInstr *, unsigned> PressureMap;
- computeSGPRPressure(MBB, PressureMap);
-
- SmallVector<Candidate, 16> Candidates;
- for (MachineInstr *MI : EligibleMIs)
- Candidates.push_back({MI, PressureMap.lookup(MI)});
-
- // Prefer highest SGPR pressure; each conversion costs one vreg_64.
- const unsigned MaxConverts = SAddrToVAddrMaxConverts > 0
- ? SAddrToVAddrMaxConverts.getValue()
- : MaxAddrVGPRsPerBB / 2;
- DenseSet<MachineInstr *> Allowed;
- llvm::sort(Candidates, [](const Candidate &A, const Candidate &B) {
- return A.Pressure > B.Pressure;
- });
- const unsigned NumToConvert =
- std::min(static_cast<unsigned>(Candidates.size()), MaxConverts);
- for (unsigned I = 0; I < NumToConvert; ++I)
- Allowed.insert(Candidates[I].MI);
-
SmallVector<Register, 8> GroupRegs;
MachineInstr *GroupLast = nullptr;
bool Changed = false;
@@ -320,7 +238,10 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
};
for (MachineInstr &MI : MBB) {
- if (!Allowed.count(&MI))
+ int SAddrIdx;
+ if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ continue;
+ if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
continue;
LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
@@ -345,19 +266,6 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
- CurMF = &MF;
-
- const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
- unsigned Occupancy = MFI->getOccupancy();
- unsigned DynBlockSize = MFI->getDynamicVGPRBlockSize();
- unsigned VGPRsAtOccupancy = ST.getMaxNumVGPRs(Occupancy, DynBlockSize);
-
- // Cap address VGPRs per BB to half the per-wave budget at target occupancy.
- MaxAddrVGPRsPerBB = VGPRsAtOccupancy / 2;
-
- LLVM_DEBUG(dbgs() << "SAddrToVAddr: occupancy=" << Occupancy
- << " VGPRsAtOcc=" << VGPRsAtOccupancy
- << " addrBudget=" << MaxAddrVGPRsPerBB << " VGPRs\n");
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
@@ -365,4 +273,3 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
return Changed;
}
-
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
deleted file mode 100644
index 8f5c706d7504e..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr-budget.mir
+++ /dev/null
@@ -1,52 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
-# RUN: -amdgpu-global-load-saddr-to-vaddr-max-converts=1 -o - %s \
-# RUN: | FileCheck -check-prefix=CAP1 %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr \
-# RUN: -o - %s | FileCheck -check-prefix=NOBUDGET %s
-
-# Test: With max-converts=1, only the highest SGPR-pressure candidate is
-# converted. Candidate 1 has SGPR pressure 4 (sgpr44, sgpr45, sgpr46, sgpr47
-# all live), candidate 2 has pressure 2 (only sgpr44, sgpr45 live).
-# The pass picks candidate 1 for conversion and leaves candidate 2 as SADDR.
-# Without the budget cap, both are converted.
----
-name: budget_cap_picks_highest_pressure
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
-
- ; CAP1-LABEL: name: budget_cap_picks_highest_pressure
- ; CAP1: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
- ; CAP1-NEXT: {{ $}}
- ; CAP1-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CAP1-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CAP1-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
- ; CAP1-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
- ; CAP1-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CAP1-NEXT: $sgpr44 = S_MOV_B32 1
- ; CAP1-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- ;
- ; NOBUDGET-LABEL: name: budget_cap_picks_highest_pressure
- ; NOBUDGET: liveins: $sgpr44_sgpr45, $sgpr46, $sgpr47
- ; NOBUDGET-NEXT: {{ $}}
- ; NOBUDGET-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; NOBUDGET-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 $sgpr46
- ; NOBUDGET-NEXT: $sgpr45 = S_MOV_B32 $sgpr47
- ; NOBUDGET-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; NOBUDGET-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; NOBUDGET-NEXT: KILL killed [[COPY]], killed [[COPY1]]
- ; NOBUDGET-NEXT: $sgpr44 = S_MOV_B32 1
- ; NOBUDGET-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 $sgpr46
- $sgpr45 = S_MOV_B32 $sgpr47
- %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 1
- S_ENDPGM 0, implicit %1, implicit %2
-...
>From 28b6d9c89bbdcbd2b964de0dfd0b2087945d7062 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 10:35:01 -0500
Subject: [PATCH 07/16] changes
---
.../AMDGPU/SIGlobalLoadSAddrToVAddr.cpp | 148 ++++++++----------
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 20 +--
2 files changed, 76 insertions(+), 92 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
index 64212ede8b3c2..d33396422264d 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
@@ -5,11 +5,12 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//
-/// \file Convert global SADDR memory ops to VADDR when the SGPR address is
-/// overwritten soon after, avoiding s_wait_xcnt hazards from replay of loads
-/// that share a scratch SGPR pair. Runs after SGPR RA and SILowerSGPRSpills.
-//
+/// \file
+/// Convert FLAT global SADDR memory ops to VADDR when the physical SGPR
+/// address pair is redefined later in the block. This avoids s_wait_xcnt
+/// stalls from XNACK replay of loads that share a reused SGPR pair.
+///
+/// Runs after SGPR RA / SILowerSGPRSpills and before VGPR RA.
//===----------------------------------------------------------------------===//
#include "SIGlobalLoadSAddrToVAddr.h"
@@ -18,6 +19,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -27,14 +29,18 @@ using namespace llvm;
#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
-static cl::opt<unsigned> SAddrToVAddrWindow(
- "amdgpu-global-load-saddr-to-vaddr-window",
- cl::desc("Instruction window to scan for saddr redefinition"), cl::init(4),
- cl::Hidden);
+STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
-static cl::opt<unsigned> SAddrToVAddrKillGroupSize(
- "amdgpu-global-load-saddr-to-vaddr-kill-group-size",
- cl::desc("Max converted loads per KILL group"), cl::init(16), cl::Hidden);
+static cl::opt<unsigned> SAddrRedefWindow(
+ "amdgpu-saddr-redef-window",
+ cl::desc("Max non-meta instructions to scan for saddr redef "
+ "(0 = scan to end of block)"),
+ cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> AddrLivenessGroupSize(
+ "amdgpu-saddr-vaddr-liveness-group",
+ cl::desc("Max converted VADDR addresses kept simultaneously live"),
+ cl::init(16), cl::Hidden);
namespace {
@@ -43,9 +49,9 @@ class SIGlobalLoadSAddrToVAddr {
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ bool sAddrRedefinedAfter(MachineInstr &MI, Register SAddr);
bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
- bool isSAddrRedefinedInWindow(MachineInstr &MI, Register SAddr);
bool processMBB(MachineBasicBlock &MBB);
public:
@@ -55,14 +61,12 @@ class SIGlobalLoadSAddrToVAddr {
class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
public:
static char ID;
-
SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- SIGlobalLoadSAddrToVAddr Impl;
- return Impl.run(MF);
+ return SIGlobalLoadSAddrToVAddr().run(MF);
}
StringRef getPassName() const override {
@@ -90,51 +94,43 @@ SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- SIGlobalLoadSAddrToVAddr Impl;
- if (!Impl.run(MF))
+ if (!SIGlobalLoadSAddrToVAddr().run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
return PA;
}
-bool SIGlobalLoadSAddrToVAddr::isSAddrRedefinedInWindow(MachineInstr &MI,
- Register SAddr) {
+/// Return true if \p MI is a FLAT global op with a physical SGPR saddr.
+static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
+ const SIRegisterInfo &TRI,
+ const MachineRegisterInfo &MRI, int &SAddrIdx) {
+ SAddrIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ return false;
+ const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ return SAddr.isReg() && SAddr.getReg().isPhysical() &&
+ TRI.isSGPRReg(MRI, SAddr.getReg());
+}
+
+bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
+ Register SAddr) {
unsigned NumScanned = 0;
- auto I = std::next(MI.getIterator());
- auto E = MI.getParent()->end();
+ MachineBasicBlock::iterator I = std::next(MI.getIterator());
+ MachineBasicBlock::iterator E = MI.getParent()->end();
for (; I != E; ++I) {
- if (I->isMetaInstruction())
- continue;
-
- ++NumScanned;
-
- for (const MachineOperand &MO : I->operands()) {
+ for (const MachineOperand &MO : I->operands())
if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
TRI->regsOverlap(MO.getReg(), SAddr))
return true;
- }
- if (NumScanned >= SAddrToVAddrWindow)
+ if (!I->isMetaInstruction() && SAddrRedefWindow > 0 &&
+ ++NumScanned >= SAddrRedefWindow)
return false;
}
return false;
}
-static bool isGlobalWithPhysSAddr(const MachineInstr &MI,
- const SIInstrInfo &TII,
- const SIRegisterInfo &TRI,
- const MachineRegisterInfo &MRI,
- int &SAddrIdx) {
- unsigned Opc = MI.getOpcode();
- SAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
- if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
- return false;
- const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
- return SAddr.isReg() && SAddr.getReg().isPhysical() &&
- TRI.isSGPRReg(MRI, SAddr.getReg());
-}
-
bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
@@ -142,9 +138,12 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
if (NewOpc < 0)
return false;
+ if (AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr) < 0)
+ return false;
- int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
- if (NewVAddrIdx < 0)
+ // VADDR form does not support scale_offset.
+ int CPolIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::cpol);
+ if (CPolIdx >= 0 && (MI.getOperand(CPolIdx).getImm() & AMDGPU::CPol::SCAL))
return false;
int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
@@ -159,12 +158,10 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
bool VAddrIsZero = false;
if (OldVAddrReg.isVirtual()) {
- MachineInstr *VAddrDef = MRI->getVRegDef(OldVAddrReg);
- if (VAddrDef &&
- (VAddrDef->isImplicitDef() ||
- (VAddrDef->isMoveImmediate() && VAddrDef->getOperand(1).isImm() &&
- VAddrDef->getOperand(1).getImm() == 0)))
- VAddrIsZero = true;
+ if (auto *Def = MRI->getVRegDef(OldVAddrReg))
+ VAddrIsZero = Def->isImplicitDef() ||
+ (Def->isMoveImmediate() && Def->getOperand(1).isImm() &&
+ Def->getOperand(1).getImm() == 0);
}
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
@@ -172,29 +169,24 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
if (VAddrIsZero) {
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
} else {
- assert(OldVAddrReg.isVirtual() &&
- TRI->getRegSizeInBits(*MRI->getRegClass(OldVAddrReg)) == 32 &&
- "Non-zero vaddr path expects a 32-bit VGPR operand");
+ MCRegister Lo = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
+ MCRegister Hi = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
- MCRegister SAddrLoPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
- MCRegister SAddrHiPhys = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
-
- Register VAddrHiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), VAddrHiExt)
+ Register HiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
.addImm(31)
.addReg(OldVAddrReg);
- Register SAddrLo = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- Register SAddrHi = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrLo).addReg(SAddrLoPhys);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrHi).addReg(SAddrHiPhys);
+ Register LoV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register HiV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV).addReg(Lo);
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV).addReg(Hi);
MCRegister VCC = TRI->getVCC();
-
auto AddLo =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), NewVAddr)
.addDef(VCC)
- .addReg(SAddrLo)
+ .addReg(LoV)
.addReg(OldVAddrReg)
.addImm(0);
AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
@@ -203,8 +195,8 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
auto AddHi =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
.addDef(VCC, RegState::Dead)
- .addReg(SAddrHi)
- .addReg(VAddrHiExt)
+ .addReg(HiV)
+ .addReg(HiExt)
.addReg(VCC, RegState::Kill)
.addImm(0);
AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
@@ -214,8 +206,9 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
VAddr.setReg(NewVAddr);
MI.removeOperand(OldSAddrIdx);
- LLVM_DEBUG(dbgs() << " Converted to VADDR: " << MI);
+ LLVM_DEBUG(dbgs() << " Converted: " << MI);
NewAddrReg = NewVAddr;
+ ++NumConverted;
return true;
}
@@ -224,12 +217,11 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
MachineInstr *GroupLast = nullptr;
bool Changed = false;
- // KILL extends address VGPR liveness to prevent RA reuse within a group.
auto FlushGroup = [&]() {
if (GroupLast && GroupRegs.size() > 1) {
- auto InsertPt = std::next(GroupLast->getIterator());
- auto KillMI = BuildMI(MBB, InsertPt, GroupLast->getDebugLoc(),
- TII->get(TargetOpcode::KILL));
+ auto KillMI =
+ BuildMI(MBB, std::next(GroupLast->getIterator()),
+ GroupLast->getDebugLoc(), TII->get(TargetOpcode::KILL));
for (Register R : GroupRegs)
KillMI.addReg(R, RegState::Kill);
}
@@ -239,20 +231,19 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
for (MachineInstr &MI : MBB) {
int SAddrIdx;
- if (!isGlobalWithPhysSAddr(MI, *TII, *TRI, *MRI, SAddrIdx))
+ if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
continue;
- if (!isSAddrRedefinedInWindow(MI, MI.getOperand(SAddrIdx).getReg()))
+ if (!sAddrRedefinedAfter(MI, MI.getOperand(SAddrIdx).getReg()))
continue;
- LLVM_DEBUG(dbgs() << " SAddr redef in window: " << MI);
Register NewAddr;
if (convertToVAddr(MI, MBB, NewAddr)) {
GroupRegs.push_back(NewAddr);
GroupLast = &MI;
Changed = true;
- if (SAddrToVAddrKillGroupSize > 0 &&
- GroupRegs.size() >= SAddrToVAddrKillGroupSize)
+ if (AddrLivenessGroupSize > 0 &&
+ GroupRegs.size() >= AddrLivenessGroupSize)
FlushGroup();
}
}
@@ -270,6 +261,5 @@ bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
Changed |= processMBB(MBB);
-
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index f05d7968c3838..54b52520fe91e 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -2,7 +2,6 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
-# Test: Basic conversion - saddr redefined 1 instruction after load
---
name: basic_convert
tracksRegLiveness: true
@@ -32,28 +31,29 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: No conversion - saddr not redefined within window (5 nops > window=4)
+# Default window=0 scans to end of block; redef after 5 NOPs still triggers.
---
-name: no_redef_no_convert
+name: redef_after_nops
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: no_redef_no_convert
+ ; CONVERT-LABEL: name: redef_after_nops
; CONVERT: liveins: $sgpr44_sgpr45
; CONVERT-NEXT: {{ $}}
; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: S_NOP 0
; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: no_redef_no_convert
+ ; DISABLED-LABEL: name: redef_after_nops
; DISABLED: liveins: $sgpr44_sgpr45
; DISABLED-NEXT: {{ $}}
; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -76,7 +76,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Store conversion
---
name: store_convert
tracksRegLiveness: true
@@ -109,7 +108,6 @@ body: |
S_ENDPGM 0
...
-# Test: Non-zero vaddr offset requires full add sequence
---
name: nonzero_vaddr
tracksRegLiveness: true
@@ -143,7 +141,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Test: Multiple converted loads get a KILL to extend address live ranges
---
name: multi_load_kill
tracksRegLiveness: true
@@ -183,6 +180,3 @@ body: |
$sgpr44 = S_MOV_B32 1
S_ENDPGM 0, implicit %1, implicit %2
...
-## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-# CONVERT: {{.*}}
-# DISABLED: {{.*}}
>From f425c50bcd47f6d832e52167b52fea589fc2b2d5 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 13:48:01 -0500
Subject: [PATCH 08/16] [AMDGPU] Rewrite SADDR memory ops to VADDR reduce
xcnt stalls
When the SGPR register allocator reuses the same physical pair across
consecutive SADDR global loads, s_wait_xcnt stalls are introduced for
XNACK replay.
Add SIFixXcntStallSAddrReuse, a post-SGPR-RA pass that converts
SADDR global memory ops to VADDR form . A KILL pseudo extends
address VGPR liveness in groups of 16 to prevent the VGPR allocator
from reintroducing the same hazard. The pass scans to the end of each
block by default and skips scale_offset loads that have no VADDR
equivalent.
Enabled by default on targets with s_wait_xcnt.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 2 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 12 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 2 +-
...VAddr.cpp => SIFixXcntStallSAddrReuse.cpp} | 45 ++--
...drToVAddr.h => SIFixXcntStallSAddrReuse.h} | 12 +-
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 201 +++++++-----------
7 files changed, 117 insertions(+), 161 deletions(-)
rename llvm/lib/Target/AMDGPU/{SIGlobalLoadSAddrToVAddr.cpp => SIFixXcntStallSAddrReuse.cpp} (86%)
rename llvm/lib/Target/AMDGPU/{SIGlobalLoadSAddrToVAddr.h => SIFixXcntStallSAddrReuse.h} (60%)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 16b3d423a45ec..4370703120abc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -214,8 +214,8 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
-void initializeSIGlobalLoadSAddrToVAddrLegacyPass(PassRegistry &);
-extern char &SIGlobalLoadSAddrToVAddrLegacyID;
+void initializeSIFixXcntStallSAddrReuseLegacyPass(PassRegistry &);
+extern char &SIFixXcntStallSAddrReuseLegacyID;
void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
extern char &SILoadStoreOptimizerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 84efd6452e008..3af2204d83281 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -133,7 +133,7 @@ MACHINE_FUNCTION_PASS("si-fix-vgpr-copies", SIFixVGPRCopiesPass())
MACHINE_FUNCTION_PASS("si-fold-operands", SIFoldOperandsPass());
MACHINE_FUNCTION_PASS("si-form-memory-clauses", SIFormMemoryClausesPass())
MACHINE_FUNCTION_PASS("si-i1-copies", SILowerI1CopiesPass())
-MACHINE_FUNCTION_PASS("si-global-load-saddr-to-vaddr", SIGlobalLoadSAddrToVAddrPass())
+MACHINE_FUNCTION_PASS("si-fix-xcnt-stall-saddr-reuse", SIFixXcntStallSAddrReusePass())
MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 6a986a16a37eb..fc8edd52557e6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -50,9 +50,9 @@
#include "R600TargetMachine.h"
#include "SIFixSGPRCopies.h"
#include "SIFixVGPRCopies.h"
+#include "SIFixXcntStallSAddrReuse.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
-#include "SIGlobalLoadSAddrToVAddr.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -728,7 +728,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
- initializeSIGlobalLoadSAddrToVAddrLegacyPass(*PR);
+ initializeSIFixXcntStallSAddrReuseLegacyPass(*PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -1816,7 +1816,7 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1855,7 +1855,7 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIGlobalLoadSAddrToVAddrLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -2461,7 +2461,7 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2557,7 +2557,7 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIGlobalLoadSAddrToVAddrPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 1b193a4b581fb..eb11087991ce4 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -161,7 +161,7 @@ add_llvm_target(AMDGPUCodeGen
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
SIFrameLowering.cpp
- SIGlobalLoadSAddrToVAddr.cpp
+ SIFixXcntStallSAddrReuse.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
SIInstrInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
similarity index 86%
rename from llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
rename to llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index d33396422264d..e9987d6b06bcc 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -1,4 +1,4 @@
-//===- SIGlobalLoadSAddrToVAddr.cpp - SADDR global loads to VADDR ---------===//
+//===- SIFixXcntStallSAddrReuse.cpp - Fix xcnt stalls from SADDR reuse ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -10,10 +10,10 @@
/// address pair is redefined later in the block. This avoids s_wait_xcnt
/// stalls from XNACK replay of loads that share a reused SGPR pair.
///
-/// Runs after SGPR RA / SILowerSGPRSpills and before VGPR RA.
+/// Runs after SGPR RA and before VGPR RA.
//===----------------------------------------------------------------------===//
-#include "SIGlobalLoadSAddrToVAddr.h"
+#include "SIFixXcntStallSAddrReuse.h"
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -27,24 +27,25 @@
using namespace llvm;
-#define DEBUG_TYPE "si-global-load-saddr-to-vaddr"
+#define DEBUG_TYPE "si-fix-xcnt-stall-saddr-reuse"
STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
static cl::opt<unsigned> SAddrRedefWindow(
"amdgpu-saddr-redef-window",
- cl::desc("Max non-meta instructions to scan for saddr redef "
+ cl::desc("Maximum instructions to scan for saddr redefines "
"(0 = scan to end of block)"),
cl::init(0), cl::Hidden);
static cl::opt<unsigned> AddrLivenessGroupSize(
"amdgpu-saddr-vaddr-liveness-group",
- cl::desc("Max converted VADDR addresses kept simultaneously live"),
+ cl::desc("Maximum converted VADDRs kept simultaneously live, kill is "
+ "issued for each group"),
cl::init(16), cl::Hidden);
namespace {
-class SIGlobalLoadSAddrToVAddr {
+class SIFixXcntStallSAddrReuse {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
@@ -58,19 +59,19 @@ class SIGlobalLoadSAddrToVAddr {
bool run(MachineFunction &MF);
};
-class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
+class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
public:
static char ID;
- SIGlobalLoadSAddrToVAddrLegacy() : MachineFunctionPass(ID) {}
+ SIFixXcntStallSAddrReuseLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- return SIGlobalLoadSAddrToVAddr().run(MF);
+ return SIFixXcntStallSAddrReuse().run(MF);
}
StringRef getPassName() const override {
- return "SI Global Load SAddr to VAddr";
+ return "SI Fix Xcnt Stall SAddr Reuse";
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -81,20 +82,20 @@ class SIGlobalLoadSAddrToVAddrLegacy : public MachineFunctionPass {
} // end anonymous namespace
-char SIGlobalLoadSAddrToVAddrLegacy::ID = 0;
+char SIFixXcntStallSAddrReuseLegacy::ID = 0;
-INITIALIZE_PASS(SIGlobalLoadSAddrToVAddrLegacy, DEBUG_TYPE,
- "SI Global Load SAddr to VAddr", false, false)
+INITIALIZE_PASS(SIFixXcntStallSAddrReuseLegacy, DEBUG_TYPE,
+ "SI Fix Xcnt Stall SAddr Reuse", false, false)
-char &llvm::SIGlobalLoadSAddrToVAddrLegacyID =
- SIGlobalLoadSAddrToVAddrLegacy::ID;
+char &llvm::SIFixXcntStallSAddrReuseLegacyID =
+ SIFixXcntStallSAddrReuseLegacy::ID;
PreservedAnalyses
-SIGlobalLoadSAddrToVAddrPass::run(MachineFunction &MF,
+SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- if (!SIGlobalLoadSAddrToVAddr().run(MF))
+ if (!SIFixXcntStallSAddrReuse().run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
@@ -113,7 +114,7 @@ static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
TRI.isSGPRReg(MRI, SAddr.getReg());
}
-bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
+bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
Register SAddr) {
unsigned NumScanned = 0;
MachineBasicBlock::iterator I = std::next(MI.getIterator());
@@ -131,7 +132,7 @@ bool SIGlobalLoadSAddrToVAddr::sAddrRedefinedAfter(MachineInstr &MI,
return false;
}
-bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
+bool SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
unsigned Opc = MI.getOpcode();
@@ -212,7 +213,7 @@ bool SIGlobalLoadSAddrToVAddr::convertToVAddr(MachineInstr &MI,
return true;
}
-bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
+bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
SmallVector<Register, 8> GroupRegs;
MachineInstr *GroupLast = nullptr;
bool Changed = false;
@@ -252,7 +253,7 @@ bool SIGlobalLoadSAddrToVAddr::processMBB(MachineBasicBlock &MBB) {
return Changed;
}
-bool SIGlobalLoadSAddrToVAddr::run(MachineFunction &MF) {
+bool SIFixXcntStallSAddrReuse::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
diff --git a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
similarity index 60%
rename from llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
rename to llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
index b89815683dfc5..9419fef056b22 100644
--- a/llvm/lib/Target/AMDGPU/SIGlobalLoadSAddrToVAddr.h
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.h
@@ -1,4 +1,4 @@
-//===- SIGlobalLoadSAddrToVAddr.h -------------------------------*- C++ -*-===//
+//===- SIFixXcntStallSAddrReuse.h -------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,15 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
-#define LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#ifndef LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
+#define LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
#include "llvm/CodeGen/MachinePassManager.h"
namespace llvm {
-class SIGlobalLoadSAddrToVAddrPass
- : public PassInfoMixin<SIGlobalLoadSAddrToVAddrPass> {
+class SIFixXcntStallSAddrReusePass
+ : public PassInfoMixin<SIFixXcntStallSAddrReusePass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
@@ -22,4 +22,4 @@ class SIGlobalLoadSAddrToVAddrPass
} // namespace llvm
-#endif // LLVM_LIB_TARGET_AMDGPU_SIGLOBALLOADSADDRTOVADDR_H
+#endif // LLVM_LIB_TARGET_AMDGPU_SIFIXXCNTSTALLSADDRREUSE_H
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 54b52520fe91e..a1e157cf50b7f 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,106 +1,61 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=CONVERT %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-global-load-saddr-to-vaddr -o - %s | FileCheck -check-prefix=DISABLED %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
---
-name: basic_convert
+name: global_sadd_load
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: basic_convert
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: global_sadd_load
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: basic_convert
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; GFX942-LABEL: name: global_sadd_load
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
S_ENDPGM 0, implicit %1
...
-# Default window=0 scans to end of block; redef after 5 NOPs still triggers.
---
-name: redef_after_nops
+name: global_sadd_store
tracksRegLiveness: true
body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: redef_after_nops
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: S_NOP 0
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: global_sadd_store
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0
;
- ; DISABLED-LABEL: name: redef_after_nops
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: S_NOP 0
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- S_NOP 0
- S_NOP 0
- S_NOP 0
- S_NOP 0
- S_NOP 0
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
-...
-
----
-name: store_convert
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $sgpr44_sgpr45
-
- ; CONVERT-LABEL: name: store_convert
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0
- ;
- ; DISABLED-LABEL: name: store_convert
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0
+ ; GFX942-LABEL: name: global_sadd_store
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
GLOBAL_STORE_DWORD_SADDR %0, %1, $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
@@ -115,26 +70,26 @@ body: |
bb.0:
liveins: $sgpr44_sgpr45, $vgpr10
- ; CONVERT-LABEL: name: nonzero_vaddr
- ; CONVERT: liveins: $sgpr44_sgpr45, $vgpr10
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; CONVERT-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
- ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
- ; CONVERT-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
- ; CONVERT-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
- ; CONVERT-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-LABEL: name: nonzero_vaddr
+ ; GFX1250: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
+ ; GFX1250-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
+ ; GFX1250-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
- ; DISABLED-LABEL: name: nonzero_vaddr
- ; DISABLED: liveins: $sgpr44_sgpr45, $vgpr10
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; GFX942-LABEL: name: nonzero_vaddr
+ ; GFX942: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = COPY $vgpr10
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
@@ -148,30 +103,30 @@ body: |
bb.0:
liveins: $sgpr44_sgpr45
- ; CONVERT-LABEL: name: multi_load_kill
- ; CONVERT: liveins: $sgpr44_sgpr45
- ; CONVERT-NEXT: {{ $}}
- ; CONVERT-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; CONVERT-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 0
- ; CONVERT-NEXT: $sgpr45 = S_MOV_B32 0
- ; CONVERT-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; CONVERT-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; CONVERT-NEXT: KILL killed [[COPY]], killed [[COPY1]]
- ; CONVERT-NEXT: $sgpr44 = S_MOV_B32 1
- ; CONVERT-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ ; GFX1250-LABEL: name: multi_load_kill
+ ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: KILL killed [[COPY]], killed [[COPY1]]
+ ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
- ; DISABLED-LABEL: name: multi_load_kill
- ; DISABLED: liveins: $sgpr44_sgpr45
- ; DISABLED-NEXT: {{ $}}
- ; DISABLED-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 0
- ; DISABLED-NEXT: $sgpr45 = S_MOV_B32 0
- ; DISABLED-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; DISABLED-NEXT: $sgpr44 = S_MOV_B32 1
- ; DISABLED-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ ; GFX942-LABEL: name: multi_load_kill
+ ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: $sgpr45 = S_MOV_B32 0
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: $sgpr44 = S_MOV_B32 1
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
>From 876dcbe94705398cedae0d13944419958fd81db6 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 17:51:23 -0500
Subject: [PATCH 09/16] fix tests
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 16 +-
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 100 +++++++--
.../AMDGPU/GlobalISel/load-constant.96.ll | 10 +-
.../AMDGPU/insert_vector_elt.v2bf16.ll | 43 +++-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 3 +
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 159 +++++++++-----
.../test/CodeGen/AMDGPU/loop-prefetch-data.ll | 24 ++-
.../promote-constOffset-to-imm-gfx12.ll | 10 +-
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 21 +-
.../wait-xcnt-atomic-rmw-optimization.ll | 204 +++++++++++++-----
.../waitcnt-loop-ds-prefetch-pattern.ll | 29 +--
11 files changed, 423 insertions(+), 196 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index fc8edd52557e6..b8ecb59f7ece7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1816,8 +1816,6 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIFixXcntStallSAddrReuseLegacyID);
-
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1827,6 +1825,8 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
addPass(&SILowerWWMCopiesLegacyID);
addPass(&AMDGPUReserveWWMRegsLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
+
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(false));
@@ -1855,8 +1855,6 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// Equivalent of PEI for SGPRs.
addPass(&SILowerSGPRSpillsLegacyID);
- addPass(&SIFixXcntStallSAddrReuseLegacyID);
-
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addPass(&SIPreAllocateWWMRegsLegacyID);
@@ -1866,6 +1864,8 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
addPass(createVirtRegRewriter(false));
addPass(&AMDGPUReserveWWMRegsLegacyID);
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
+
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(true));
@@ -2461,8 +2461,6 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
-
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2476,6 +2474,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
addMachineFunctionPass(SILowerWWMCopiesPass(), PMW);
addMachineFunctionPass(AMDGPUReserveWWMRegsPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
+
// VGPR allocation - default to fast at -O0.
if (VGPRRegAllocNPM == RegAllocType::Greedy)
addMachineFunctionPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
@@ -2557,8 +2557,6 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
// Equivalent of PEI for SGPRs.
addMachineFunctionPass(SILowerSGPRSpillsPass(), PMW);
- addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
-
// To Allocate wwm registers used in whole quad mode operations (for shaders).
addMachineFunctionPass(SIPreAllocateWWMRegsPass(), PMW);
@@ -2572,6 +2570,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
addMachineFunctionPass(VirtRegRewriterPass(false), PMW);
addMachineFunctionPass(AMDGPUReserveWWMRegsPass(), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
+
// VGPR allocation - default to greedy at -O1 and above.
if (VGPRRegAllocNPM == RegAllocType::Fast)
addMachineFunctionPass(RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index e9987d6b06bcc..3bea1e613fc1e 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -20,6 +20,7 @@
#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
@@ -49,13 +50,15 @@ class SIFixXcntStallSAddrReuse {
const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
+ LiveIntervals *LIS = nullptr;
bool sAddrRedefinedAfter(MachineInstr &MI, Register SAddr);
- bool convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
- Register &NewAddrReg);
+ MachineInstr *convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
+ Register &NewAddrReg);
bool processMBB(MachineBasicBlock &MBB);
public:
+ SIFixXcntStallSAddrReuse(LiveIntervals *LIS = nullptr) : LIS(LIS) {}
bool run(MachineFunction &MF);
};
@@ -67,7 +70,9 @@ class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
bool runOnMachineFunction(MachineFunction &MF) override {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- return SIFixXcntStallSAddrReuse().run(MF);
+ auto *LISWrapper = getAnalysisIfAvailable<llvm::LiveIntervalsWrapperPass>();
+ llvm::LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
+ return SIFixXcntStallSAddrReuse(LIS).run(MF);
}
StringRef getPassName() const override {
@@ -75,7 +80,7 @@ class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
+ AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
@@ -95,10 +100,13 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- if (!SIFixXcntStallSAddrReuse().run(MF))
+ auto *LIS = MFAM.getCachedResult<llvm::LiveIntervalsAnalysis>(MF);
+ if (!SIFixXcntStallSAddrReuse(LIS).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
+ if (LIS)
+ PA.preserve<llvm::LiveIntervalsAnalysis>();
return PA;
}
@@ -132,20 +140,20 @@ bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
return false;
}
-bool SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
+MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
MachineBasicBlock &MBB,
Register &NewAddrReg) {
unsigned Opc = MI.getOpcode();
int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
if (NewOpc < 0)
- return false;
+ return nullptr;
if (AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr) < 0)
- return false;
+ return nullptr;
// VADDR form does not support scale_offset.
int CPolIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::cpol);
if (CPolIdx >= 0 && (MI.getOperand(CPolIdx).getImm() & AMDGPU::CPol::SCAL))
- return false;
+ return nullptr;
int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
@@ -168,30 +176,47 @@ bool SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
if (VAddrIsZero) {
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
+ auto Copy = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*Copy);
} else {
MCRegister Lo = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
MCRegister Hi = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
+ Register TmpVAddr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ auto CopyVAddr = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), TmpVAddr).add(VAddr);
+ if (LIS) {
+ LIS->InsertMachineInstrInMaps(*CopyVAddr);
+ LIS->createAndComputeVirtRegInterval(TmpVAddr);
+ }
+
Register HiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
+ auto Ashr = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
.addImm(31)
- .addReg(OldVAddrReg);
+ .addReg(TmpVAddr);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*Ashr);
Register LoV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register HiV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV).addReg(Lo);
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV).addReg(Hi);
+ auto CopyLo = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV).addReg(Lo);
+ auto CopyHi = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV).addReg(Hi);
+ if (LIS) {
+ LIS->InsertMachineInstrInMaps(*CopyLo);
+ LIS->InsertMachineInstrInMaps(*CopyHi);
+ }
MCRegister VCC = TRI->getVCC();
auto AddLo =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), NewVAddr)
.addDef(VCC)
.addReg(LoV)
- .addReg(OldVAddrReg)
+ .addReg(TmpVAddr)
.addImm(0);
AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
AddLo->getOperand(0).setIsUndef(true);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*AddLo);
auto AddHi =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
@@ -201,16 +226,41 @@ bool SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
.addReg(VCC, RegState::Kill)
.addImm(0);
AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*AddHi);
+
+ if (LIS) {
+ LIS->createAndComputeVirtRegInterval(HiExt);
+ LIS->createAndComputeVirtRegInterval(LoV);
+ LIS->createAndComputeVirtRegInterval(HiV);
+ }
}
- MI.setDesc(TII->get(NewOpc));
- VAddr.setReg(NewVAddr);
- MI.removeOperand(OldSAddrIdx);
+ if (LIS)
+ LIS->createAndComputeVirtRegInterval(NewVAddr);
+
+ MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(NewOpc));
+ for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
+ if (i == (unsigned)OldSAddrIdx)
+ continue;
+ if (i == (unsigned)OldVAddrIdx) {
+ MIB.addReg(NewVAddr);
+ } else {
+ MIB.add(MI.getOperand(i));
+ }
+ }
+
+ MIB.cloneMemRefs(MI);
+
+ if (LIS) {
+ LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+ }
- LLVM_DEBUG(dbgs() << " Converted: " << MI);
+ LLVM_DEBUG(dbgs() << " Converted: " << *MIB);
+ MI.eraseFromParent();
NewAddrReg = NewVAddr;
++NumConverted;
- return true;
+ return MIB.getInstr();
}
bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
@@ -225,12 +275,15 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
GroupLast->getDebugLoc(), TII->get(TargetOpcode::KILL));
for (Register R : GroupRegs)
KillMI.addReg(R, RegState::Kill);
+ if (LIS) {
+ LIS->InsertMachineInstrInMaps(*KillMI);
+ }
}
GroupRegs.clear();
GroupLast = nullptr;
};
- for (MachineInstr &MI : MBB) {
+ for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
int SAddrIdx;
if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
continue;
@@ -238,9 +291,10 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
continue;
Register NewAddr;
- if (convertToVAddr(MI, MBB, NewAddr)) {
+ MachineInstr *NewMI = convertToVAddr(MI, MBB, NewAddr);
+ if (NewMI) {
GroupRegs.push_back(NewAddr);
- GroupLast = &MI;
+ GroupLast = NewMI;
Changed = true;
if (AddrLivenessGroupSize > 0 &&
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 96568e3c9e152..63dd2d182ae5b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -922,8 +922,9 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align1(ptr addrspace(4) inreg
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align1:
; GFX1250-UNALIGNED: ; %bb.0:
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v0, s[0:1] nv
+; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-UNALIGNED-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off nv
; GFX1250-UNALIGNED-NEXT: s_wait_loadcnt 0x0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s1, v1
@@ -1202,8 +1203,9 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(ptr addrspace(4) inreg
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align2:
; GFX1250-UNALIGNED: ; %bb.0:
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v0, s[0:1] nv
+; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-UNALIGNED-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off nv
; GFX1250-UNALIGNED-NEXT: s_wait_loadcnt 0x0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s1, v1
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 3d91e82f1b357..9995493687a8e 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -2155,16 +2155,28 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
+; GFX1250-FAKE16-NEXT: ; kill: killed $sgpr2_sgpr3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
+; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0 killed $vgpr8 killed $exec
+; GFX1250-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX1250-FAKE16-NEXT: ; kill: def $vgpr4 killed $vgpr8 killed $exec
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: global_load_b128 v[0:3], v8, s[2:3]
-; GFX1250-FAKE16-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
+; GFX1250-FAKE16-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_mov_b32 v4, s2
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 6
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
+; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
+; GFX1250-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, v4, v4
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 7
+; GFX1250-FAKE16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v6, s3
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v6, v5, vcc_lo
+; GFX1250-FAKE16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
+; GFX1250-FAKE16-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr4_vgpr5
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v9, v3, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
@@ -2237,18 +2249,29 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
+; GFX1250-REAL16-NEXT: ; kill: killed $sgpr2_sgpr3
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-REAL16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr8 killed $exec
+; GFX1250-REAL16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr4 killed $vgpr8 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: s_clause 0x1
-; GFX1250-REAL16-NEXT: global_load_b128 v[0:3], v8, s[2:3]
-; GFX1250-REAL16-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
+; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_mov_b32 v4, s2
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v6, s3
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 6
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-REAL16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
+; GFX1250-REAL16-NEXT: v_add_co_u32 v4, vcc_lo, v4, v4
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v6, v5, vcc_lo
+; GFX1250-REAL16-NEXT: global_load_b128 v[0:3], v[0:1], off
; GFX1250-REAL16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 7
+; GFX1250-REAL16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
; GFX1250-REAL16-NEXT: s_cselect_b32 s3, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 4
+; GFX1250-REAL16-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr4_vgpr5
; GFX1250-REAL16-NEXT: s_cselect_b32 s6, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 5
; GFX1250-REAL16-NEXT: s_cselect_b32 s7, -1, 0
@@ -2278,6 +2301,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_cselect_b32 s5, -1, 0
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x1
; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.l, v3.l, s4, s2
+; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.h, v3.h, s4, s3
+; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.l, v2.l, s4, s6
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_cndmask_b16 v7.l, v7.l, s4, s12
; GFX1250-REAL16-NEXT: v_cndmask_b16 v7.h, v7.h, s4, s13
@@ -2287,8 +2312,6 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_cndmask_b16 v5.h, v5.h, s4, s17
; GFX1250-REAL16-NEXT: v_cndmask_b16 v4.l, v4.l, s4, s18
; GFX1250-REAL16-NEXT: v_cndmask_b16 v4.h, v4.h, s4, s5
-; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.h, v3.h, s4, s3
-; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.l, v2.l, s4, s6
; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.h, v2.h, s4, s7
; GFX1250-REAL16-NEXT: v_cndmask_b16 v1.l, v1.l, s4, s8
; GFX1250-REAL16-NEXT: v_cndmask_b16 v1.h, v1.h, s4, s9
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index c49b2b927bd31..b6453efe2085f 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,6 +68,7 @@
; GCN-O0-NEXT: regallocfast<filter=wwm;no-clear-vregs>
; GCN-O0-NEXT: si-lower-wwm-copies
; GCN-O0-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O0-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O0-NEXT: regallocfast<filter=vgpr>
; GCN-O0-NEXT: si-fix-vgpr-copies
; GCN-O0-NEXT: remove-redundant-debug-values
@@ -216,6 +217,7 @@
; GCN-O2-NEXT: si-lower-wwm-copies
; GCN-O2-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O2-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O2-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O2-NEXT: greedy<vgpr>
; GCN-O2-NEXT: amdgpu-nsa-reassign
; GCN-O2-NEXT: amdgpu-rewrite-agpr-copy-mfma
@@ -385,6 +387,7 @@
; GCN-O3-NEXT: si-lower-wwm-copies
; GCN-O3-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O3-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O3-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O3-NEXT: greedy<vgpr>
; GCN-O3-NEXT: amdgpu-nsa-reassign
; GCN-O3-NEXT: amdgpu-rewrite-agpr-copy-mfma
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 4b9fdf3a768dc..14b22b924b689 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -1272,18 +1272,18 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v3, s[2:3] nv
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 2
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
; GFX1250-REAL16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
@@ -1488,18 +1488,19 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v4, s[2:3] nv
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10002
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s5
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
; GFX1250-REAL16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
@@ -1735,12 +1736,13 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v8, 0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_u8 v0, v8, s[2:3] nv
+; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-FAKE16-NEXT: global_load_u8 v0, v[0:1], off nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX1250-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10005
@@ -1761,10 +1763,10 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v8, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v8, s[2:3] nv
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s5, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
@@ -1775,10 +1777,11 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10004
; GFX1250-REAL16-NEXT: s_lshr_b32 s9, s5, 7
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s5, 0x10006
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v2, s5
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v5, s4
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v4, s7
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s9
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v6, s8
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v7, s3
; GFX1250-REAL16-NEXT: s_clause 0x1
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
@@ -2133,10 +2136,10 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v16, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v16, s[2:3] nv
+; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: global_load_u16 v0, v[0:1], off nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
@@ -2155,14 +2158,15 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v15, s3
-; GFX1250-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s13
-; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v7, s12
-; GFX1250-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v12, s9
-; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v14, s14
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
+; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
+; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
+; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
+; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
+; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
+; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
+; GFX1250-NEXT: v_mov_b32_e32 v14, s14
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -5484,8 +5488,16 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
+; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -5495,7 +5507,6 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
@@ -5729,8 +5740,16 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
+; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -5742,7 +5761,7 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s3
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s4
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: global_store_b64 v1, v[0:1], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
@@ -6003,8 +6022,16 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-FAKE16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, s3
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
+; GFX1250-FAKE16-NEXT: global_load_u8 v0, v[2:3], off nv
+; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -6012,7 +6039,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 3, v0
; GFX1250-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s3
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s3
; GFX1250-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -6030,8 +6057,14 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
+; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -6039,7 +6072,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
; GFX1250-REAL16-NEXT: s_and_b32 s4, 0xffff, s4
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, v1
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
@@ -6599,13 +6632,13 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v16, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u8 v0, v16, s[2:3] nv
+; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: global_load_u8 v0, v[0:1], off nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v10, s3
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v10, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 6
; GFX1250-NEXT: s_lshr_b32 s4, s3, 7
; GFX1250-NEXT: s_lshr_b32 s6, s3, 4
@@ -6973,34 +7006,45 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr3 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: global_load_u16 v0, v3, s[2:3] nv
+; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_mov_b32 v0, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
+; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
+; GFX1250-REAL16-NEXT: global_load_u16 v0, v[0:1], off nv
+; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000a
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v5, v3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10009
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s3, 0x10005
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
@@ -7010,12 +7054,11 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s2
@@ -7388,13 +7431,13 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v32, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
+; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: global_load_u16 v0, v[0:1], off nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v28, s3
+; GFX1250-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v28, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index 29ff00607bd66..f3bcc19e234b1 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -236,16 +236,18 @@ define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrsp
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 0xb0
; GFX1250-NEXT: .LBB1_2: ; %for.body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: global_load_b128 v[2:5], v0, s[2:3] offset:-176
-; GFX1250-NEXT: global_prefetch_b8 v0, s[2:3] scope:SCOPE_SE
+; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: global_prefetch_b8 v[6:7], off scope:SCOPE_SE
+; GFX1250-NEXT: global_load_b128 v[2:5], v[2:3], off offset:-176
; GFX1250-NEXT: s_add_co_i32 s6, s6, -1
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 16
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b128 v0, v[2:5], s[0:1]
-; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr0_sgpr1 killed $exec
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX1250-NEXT: ; kill: killed $vgpr2_vgpr3 killed $vgpr6_vgpr7 killed $vgpr6_vgpr7
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-NEXT: s_cbranch_scc1 .LBB1_2
; GFX1250-NEXT: .LBB1_3: ; %for.end
; GFX1250-NEXT: s_endpgm
@@ -361,19 +363,21 @@ define amdgpu_kernel void @copy_constant(ptr addrspace(1) nocapture %d, ptr addr
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: .LBB2_2: ; %for.body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr2_sgpr3 killed $exec
+; GFX1250-NEXT: global_prefetch_b8 v[2:3], off offset:176 scope:SCOPE_SE
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_prefetch_b8 v0, s[2:3] offset:176 scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[8:11], s[2:3], 0x0 nv
; GFX1250-NEXT: s_add_co_i32 s6, s6, -1
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 16
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
-; GFX1250-NEXT: global_store_b128 v0, v[2:5], s[0:1]
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX1250-NEXT: ; kill: killed $vgpr2_vgpr3 killed $vgpr6_vgpr7
+; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-NEXT: s_cbranch_scc1 .LBB2_2
; GFX1250-NEXT: .LBB2_3: ; %for.end
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index a20660c844919..a66b2934b086a 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -11,12 +11,13 @@ define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %
; GFX1250-LABEL: promote_async_load_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr0_sgpr1 killed $exec
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v0, s[0:1]
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xfffffe00, 0
@@ -103,12 +104,13 @@ define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1)
; GFX1250-LABEL: promote_async_store_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr0_sgpr1 killed $exec
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
+; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v1, off
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_store_async_from_lds_b128 v0, v1, s[0:1]
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xfffffe00, 0
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index a1e157cf50b7f..9a4e2b4890220 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -14,7 +14,7 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
@@ -44,7 +44,7 @@ body: |
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec, implicit $exec :: (store (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0
;
@@ -74,12 +74,13 @@ body: |
; GFX1250: liveins: $sgpr44_sgpr45, $vgpr10
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY]], implicit $exec
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $sgpr44
- ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr45
- ; GFX1250-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY1]], [[COPY]], 0, implicit $exec
- ; GFX1250-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY2]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr44
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $sgpr45
+ ; GFX1250-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY2]], [[COPY1]], 0, implicit $exec
+ ; GFX1250-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY3]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
@@ -108,11 +109,11 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: KILL killed [[COPY]], killed [[COPY1]]
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 0f6edafb22908..38c76ba2d94e9 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -7,6 +7,8 @@ define amdgpu_kernel void @single_atomic_rmw(ptr addrspace(1) %ptr) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -32,6 +34,8 @@ define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -80,7 +84,13 @@ define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i3
; GFX1250-LABEL: atomic_rmw_with_alu:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0xc nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0xc nv
@@ -135,7 +145,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_load(ptr addrspace(1) %pt
; GFX1250-LABEL: atomic_rmw_broken_by_global_load:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -187,7 +201,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_store(ptr addrspace(1) %p
; GFX1250-LABEL: atomic_rmw_broken_by_global_store:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -242,7 +260,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr,
; GFX1250-LABEL: atomic_rmw_broken_by_flat_load:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -293,7 +315,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr
; GFX1250-LABEL: atomic_rmw_broken_by_flat_store:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -347,7 +373,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_smem_load(ptr addrspace(1) %ptr,
; GFX1250-LABEL: atomic_rmw_broken_by_smem_load:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -401,7 +431,11 @@ define amdgpu_kernel void @atomic_rmw_broken_by_atomic_store(ptr addrspace(1) %p
; GFX1250-LABEL: atomic_rmw_broken_by_atomic_store:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -462,7 +496,11 @@ define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr a
; GFX1250-LABEL: atomic_rmw_with_lds_load:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -514,7 +552,11 @@ define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr
; GFX1250-LABEL: atomic_rmw_with_lds_store:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -568,7 +610,11 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr,
; GFX1250-LABEL: atomic_rmw_with_flat_lds_load:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -593,14 +639,17 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s3, -1
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s5, -1, 0
; GFX1250-NEXT: s_mov_b64 s[6:7], 0
; GFX1250-NEXT: s_mov_b32 s4, s7
; GFX1250-NEXT: s_mov_b64 s[8:9], src_shared_base
; GFX1250-NEXT: s_mov_b32 s3, s9
-; GFX1250-NEXT: s_mov_b32 s5, -1
-; GFX1250-NEXT: s_cmp_lg_u32 s2, s5
+; GFX1250-NEXT: s_and_b32 s8, s5, exec_lo
; GFX1250-NEXT: s_cselect_b32 s4, s3, s4
; GFX1250-NEXT: s_mov_b32 s3, s6
+; GFX1250-NEXT: s_and_b32 s5, s5, exec_lo
; GFX1250-NEXT: s_cselect_b32 s2, s2, s3
; GFX1250-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GFX1250-NEXT: s_mov_b32 s3, s4
@@ -631,7 +680,11 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr,
; GFX1250-LABEL: atomic_rmw_with_flat_lds_store:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -656,14 +709,17 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr,
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s3, -1
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s5, -1, 0
; GFX1250-NEXT: s_mov_b64 s[6:7], 0
; GFX1250-NEXT: s_mov_b32 s4, s7
; GFX1250-NEXT: s_mov_b64 s[8:9], src_shared_base
; GFX1250-NEXT: s_mov_b32 s3, s9
-; GFX1250-NEXT: s_mov_b32 s5, -1
-; GFX1250-NEXT: s_cmp_lg_u32 s2, s5
+; GFX1250-NEXT: s_and_b32 s8, s5, exec_lo
; GFX1250-NEXT: s_cselect_b32 s4, s3, s4
; GFX1250-NEXT: s_mov_b32 s3, s6
+; GFX1250-NEXT: s_and_b32 s5, s5, exec_lo
; GFX1250-NEXT: s_cselect_b32 s2, s2, s3
; GFX1250-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
; GFX1250-NEXT: s_mov_b32 s3, s4
@@ -698,7 +754,13 @@ define amdgpu_kernel void @atomic_rmw_borken_by_async_lds_copy(ptr addrspace(1)
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b64 s[2:3], s[4:5]
+; GFX1250-NEXT: s_load_b64 s[0:1], s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[2:3], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[2:3], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[2:3], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[4:5], s[2:3], 0x8 nv
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -758,7 +820,11 @@ define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, pt
; GFX1250-LABEL: multiple_atomic_rmw_blocks:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
@@ -823,6 +889,8 @@ define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -883,9 +951,13 @@ define amdgpu_kernel void @atomic_rmw_across_basic_blocks(ptr addrspace(1) %ptr,
; GFX1250-LABEL: atomic_rmw_across_basic_blocks:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
@@ -917,8 +989,13 @@ define amdgpu_kernel void @atomic_rmw_across_basic_blocks(ptr addrspace(1) %ptr,
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s1, 0
-; GFX1250-NEXT: s_cmp_lg_u32 s0, s1
-; GFX1250-NEXT: s_cbranch_scc1 .LBB16_2
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_mov_b32 s1, -1
+; GFX1250-NEXT: s_xor_b32 s0, s0, s1
+; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_vccnz .LBB16_2
; GFX1250-NEXT: ; %bb.1: ; %then
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
@@ -963,62 +1040,71 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-LABEL: atomic_rmw_in_loop:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
+; GFX1250-NEXT: v_writelane_b32 v5, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s3, 1
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s1, 2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v5, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: .LBB17_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 3
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 2
-; GFX1250-NEXT: v_readlane_b32 s2, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s3, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s2, 1
; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
-; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
-; GFX1250-NEXT: s_mov_b32 s6, exec_lo
-; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: s_cmp_ge_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s1, -1, 0
+; GFX1250-NEXT: s_mov_b32 s2, -1
+; GFX1250-NEXT: s_xor_b32 s1, s1, s2
+; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s1
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
+; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
-; GFX1250-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX1250-NEXT: s_cbranch_vccnz .LBB17_1
; GFX1250-NEXT: ; %bb.2: ; %exit
; GFX1250-NEXT: s_endpgm
entry:
@@ -1043,65 +1129,70 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-LABEL: atomic_rmw_with_branch:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
-; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
+; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v5, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v5, s5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 1
+; GFX1250-NEXT: v_mov_b32_e32 v2, 1
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 2
+; GFX1250-NEXT: v_mov_b32_e32 v4, 2
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s0, -1
; GFX1250-NEXT: s_mov_b32 s2, 0
; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
; GFX1250-NEXT: .LBB18_1: ; %Flow
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 2
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX1250-NEXT: s_mov_b32 s0, 1
-; GFX1250-NEXT: v_cmp_ne_u32_e64 s0, v0, s0
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 2
+; GFX1250-NEXT: s_mov_b32 s1, -1
+; GFX1250-NEXT: s_xor_b32 s0, s0, s1
; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
; GFX1250-NEXT: ; %bb.2: ; %bb1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 3
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1117,39 +1208,40 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_branch .LBB18_4
; GFX1250-NEXT: .LBB18_3: ; %bb2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: v_mov_b32_e32 v2, 4
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_branch .LBB18_1
; GFX1250-NEXT: .LBB18_4: ; %merge
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 5
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1194,6 +1286,8 @@ define amdgpu_kernel void @atomic_rmw_fallthrough(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[2:3], s[0:1]
; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index af53cb587bc7d..7f98f8cf8de06 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -12,14 +12,14 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
; CHECK-NEXT: v_and_b32_e32 v12, 0x3ff, v0
-; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8
-; CHECK-NEXT: v_mov_b32_e32 v11, v8
+; CHECK-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v7, v4
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s1
; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: ds_load_b128 v[4:7], v13
+; CHECK-NEXT: ds_load_b128 v[8:11], v13
; CHECK-NEXT: ds_load_b128 v[0:3], v13 offset:16
; CHECK-NEXT: s_wait_dscnt 0x0
; CHECK-NEXT: .LBB0_1: ; %loop
@@ -27,23 +27,24 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_barrier_signal -1
; CHECK-NEXT: s_add_co_i32 s1, s1, 1
; CHECK-NEXT: s_wait_dscnt 0x1
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[10:11], v[6:7]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[8:9], v[4:5]
-; CHECK-NEXT: v_lshl_add_u32 v10, s1, 5, v13
+; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[10:11]
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[8:9]
+; CHECK-NEXT: v_lshl_add_u32 v14, s1, 5, v13
; CHECK-NEXT: s_cmp_lt_i32 s1, s0
; CHECK-NEXT: s_wait_dscnt 0x0
-; CHECK-NEXT: v_pk_add_f32 v[8:9], v[6:7], v[2:3]
-; CHECK-NEXT: v_pk_add_f32 v[14:15], v[4:5], v[0:1]
+; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[0:1]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[6:7]
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[4:5]
; CHECK-NEXT: s_barrier_wait -1
-; CHECK-NEXT: ds_load_b128 v[4:7], v10
-; CHECK-NEXT: ds_load_b128 v[0:3], v10 offset:16
-; CHECK-NEXT: v_pk_add_f32 v[10:11], v[8:9], v[8:9]
-; CHECK-NEXT: v_pk_add_f32 v[8:9], v[14:15], v[14:15]
+; CHECK-NEXT: ds_load_b128 v[8:11], v14
+; CHECK-NEXT: ds_load_b128 v[0:3], v14 offset:16
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: global_store_b128 v12, v[8:11], s[0:1] scale_offset
+; CHECK-NEXT: global_store_b128 v12, v[4:7], s[0:1] scale_offset
; CHECK-NEXT: s_endpgm
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
>From e98e89d16d42426e5cd70240ada138f626daf3e4 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 17:57:50 -0500
Subject: [PATCH 10/16] LIS changes
---
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 46 +----------
.../AMDGPU/insert_vector_elt.v2bf16.ll | 2 -
.../test/CodeGen/AMDGPU/loop-prefetch-data.ll | 2 -
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 1 -
.../wait-xcnt-atomic-rmw-optimization.ll | 76 +++++++++----------
5 files changed, 38 insertions(+), 89 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index 3bea1e613fc1e..be9db64cc2ad2 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -24,7 +24,6 @@
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
-#include "llvm/Support/CommandLine.h"
using namespace llvm;
@@ -32,18 +31,6 @@ using namespace llvm;
STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
-static cl::opt<unsigned> SAddrRedefWindow(
- "amdgpu-saddr-redef-window",
- cl::desc("Maximum instructions to scan for saddr redefines "
- "(0 = scan to end of block)"),
- cl::init(0), cl::Hidden);
-
-static cl::opt<unsigned> AddrLivenessGroupSize(
- "amdgpu-saddr-vaddr-liveness-group",
- cl::desc("Maximum converted VADDRs kept simultaneously live, kill is "
- "issued for each group"),
- cl::init(16), cl::Hidden);
-
namespace {
class SIFixXcntStallSAddrReuse {
@@ -124,7 +111,6 @@ static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
Register SAddr) {
- unsigned NumScanned = 0;
MachineBasicBlock::iterator I = std::next(MI.getIterator());
MachineBasicBlock::iterator E = MI.getParent()->end();
for (; I != E; ++I) {
@@ -132,10 +118,6 @@ bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
TRI->regsOverlap(MO.getReg(), SAddr))
return true;
-
- if (!I->isMetaInstruction() && SAddrRedefWindow > 0 &&
- ++NumScanned >= SAddrRedefWindow)
- return false;
}
return false;
}
@@ -264,25 +246,8 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
}
bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
- SmallVector<Register, 8> GroupRegs;
- MachineInstr *GroupLast = nullptr;
bool Changed = false;
- auto FlushGroup = [&]() {
- if (GroupLast && GroupRegs.size() > 1) {
- auto KillMI =
- BuildMI(MBB, std::next(GroupLast->getIterator()),
- GroupLast->getDebugLoc(), TII->get(TargetOpcode::KILL));
- for (Register R : GroupRegs)
- KillMI.addReg(R, RegState::Kill);
- if (LIS) {
- LIS->InsertMachineInstrInMaps(*KillMI);
- }
- }
- GroupRegs.clear();
- GroupLast = nullptr;
- };
-
for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
int SAddrIdx;
if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
@@ -291,19 +256,10 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
continue;
Register NewAddr;
- MachineInstr *NewMI = convertToVAddr(MI, MBB, NewAddr);
- if (NewMI) {
- GroupRegs.push_back(NewAddr);
- GroupLast = NewMI;
+ if (convertToVAddr(MI, MBB, NewAddr))
Changed = true;
-
- if (AddrLivenessGroupSize > 0 &&
- GroupRegs.size() >= AddrLivenessGroupSize)
- FlushGroup();
- }
}
- FlushGroup();
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 9995493687a8e..70de730d409fc 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -2176,7 +2176,6 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v6, v5, vcc_lo
; GFX1250-FAKE16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
-; GFX1250-FAKE16-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr4_vgpr5
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v9, v3, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
@@ -2271,7 +2270,6 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
; GFX1250-REAL16-NEXT: s_cselect_b32 s3, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 4
-; GFX1250-REAL16-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr4_vgpr5
; GFX1250-REAL16-NEXT: s_cselect_b32 s6, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 5
; GFX1250-REAL16-NEXT: s_cselect_b32 s7, -1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index f3bcc19e234b1..60a658d33e11b 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -245,7 +245,6 @@ define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrsp
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr0_sgpr1 killed $exec
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
-; GFX1250-NEXT: ; kill: killed $vgpr2_vgpr3 killed $vgpr6_vgpr7 killed $vgpr6_vgpr7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-NEXT: s_cbranch_scc1 .LBB1_2
@@ -376,7 +375,6 @@ define amdgpu_kernel void @copy_constant(ptr addrspace(1) nocapture %d, ptr addr
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
-; GFX1250-NEXT: ; kill: killed $vgpr2_vgpr3 killed $vgpr6_vgpr7
; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
; GFX1250-NEXT: s_cbranch_scc1 .LBB2_2
; GFX1250-NEXT: .LBB2_3: ; %for.end
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index 9a4e2b4890220..d9c234c636ec6 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -114,7 +114,6 @@ body: |
; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
- ; GFX1250-NEXT: KILL killed [[COPY]], killed [[COPY1]]
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 38c76ba2d94e9..9f07d54a0f5dc 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -1045,28 +1045,28 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
-; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
+; GFX1250-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_writelane_b32 v5, s2, 0
-; GFX1250-NEXT: v_writelane_b32 v5, s3, 1
+; GFX1250-NEXT: v_writelane_b32 v3, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v3, s3, 1
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v5, s1, 2
-; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v3, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v3, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: .LBB17_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v5, 3
-; GFX1250-NEXT: v_readlane_b32 s1, v5, 2
-; GFX1250-NEXT: v_readlane_b32 s2, v5, 0
-; GFX1250-NEXT: v_readlane_b32 s3, v5, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v3, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v3, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v3, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v3, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -1080,18 +1080,17 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v4, s0
-; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s2, 1
; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
; GFX1250-NEXT: s_cmp_ge_u32 s0, s1
@@ -1099,9 +1098,9 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_mov_b32 s2, -1
; GFX1250-NEXT: s_xor_b32 s1, s1, s2
; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s1
-; GFX1250-NEXT: v_writelane_b32 v5, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v3, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_vccnz .LBB17_1
@@ -1136,9 +1135,9 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1250-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
-; GFX1250-NEXT: v_writelane_b32 v5, s4, 0
-; GFX1250-NEXT: v_writelane_b32 v5, s5, 1
+; GFX1250-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v3, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v3, s5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v2, 1
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -1152,47 +1151,46 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v4, 2
-; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v2, 2
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[2:3], v4, off offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; kill: killed $vgpr0_vgpr1 killed $vgpr2_vgpr3
; GFX1250-NEXT: s_mov_b32 s0, -1
; GFX1250-NEXT: s_mov_b32 s2, 0
; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v3, s0, 2
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
; GFX1250-NEXT: .LBB18_1: ; %Flow
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v5, 2
+; GFX1250-NEXT: v_readlane_b32 s0, v3, 2
; GFX1250-NEXT: s_mov_b32 s1, -1
; GFX1250-NEXT: s_xor_b32 s0, s0, s1
; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
; GFX1250-NEXT: ; %bb.2: ; %bb1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 3
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1208,12 +1206,12 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_branch .LBB18_4
; GFX1250-NEXT: .LBB18_3: ; %bb2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v2, 4
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
@@ -1228,20 +1226,20 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v5, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v3, s0, 2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v5, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_branch .LBB18_1
; GFX1250-NEXT: .LBB18_4: ; %merge
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v5, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v5, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v5, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 5
; GFX1250-NEXT: s_wait_loadcnt 0x0
>From 22681b1997e9e975914fecabd5b2e9ef62b23ea3 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 18:33:29 -0500
Subject: [PATCH 11/16] minor
---
.../Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index be9db64cc2ad2..c6ff3511e7aa4 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -8,7 +8,7 @@
/// \file
/// Convert FLAT global SADDR memory ops to VADDR when the physical SGPR
/// address pair is redefined later in the block. This avoids s_wait_xcnt
-/// stalls from XNACK replay of loads that share a reused SGPR pair.
+/// stalls from XNACK replay hazards on reused SGPR pairs.
///
/// Runs after SGPR RA and before VGPR RA.
//===----------------------------------------------------------------------===//
@@ -97,7 +97,6 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
return PA;
}
-/// Return true if \p MI is a FLAT global op with a physical SGPR saddr.
static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
const SIRegisterInfo &TRI,
const MachineRegisterInfo &MRI, int &SAddrIdx) {
@@ -123,8 +122,8 @@ bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
}
MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
- MachineBasicBlock &MBB,
- Register &NewAddrReg) {
+ MachineBasicBlock &MBB,
+ Register &NewAddrReg) {
unsigned Opc = MI.getOpcode();
int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
if (NewOpc < 0)
@@ -158,7 +157,8 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
if (VAddrIsZero) {
- auto Copy = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
+ auto Copy =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
if (LIS)
LIS->InsertMachineInstrInMaps(*Copy);
} else {
@@ -166,7 +166,8 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
MCRegister Hi = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
Register TmpVAddr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- auto CopyVAddr = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), TmpVAddr).add(VAddr);
+ auto CopyVAddr =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), TmpVAddr).add(VAddr);
if (LIS) {
LIS->InsertMachineInstrInMaps(*CopyVAddr);
LIS->createAndComputeVirtRegInterval(TmpVAddr);
@@ -174,8 +175,8 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
Register HiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
auto Ashr = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
- .addImm(31)
- .addReg(TmpVAddr);
+ .addImm(31)
+ .addReg(TmpVAddr);
if (LIS)
LIS->InsertMachineInstrInMaps(*Ashr);
>From 9a6722402a1c2400985579a0919b42a65a1fb195 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Mon, 6 Apr 2026 19:24:57 -0500
Subject: [PATCH 12/16] assertion issues
---
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 6 ++--
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 ++++
.../promote-constOffset-to-imm-gfx12.ll | 28 ++++++++++++-------
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 5 ++++
4 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index c6ff3511e7aa4..450c9bf7749b9 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -87,13 +87,13 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
- auto *LIS = MFAM.getCachedResult<llvm::LiveIntervalsAnalysis>(MF);
+ auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
if (!SIFixXcntStallSAddrReuse(LIS).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
if (LIS)
- PA.preserve<llvm::LiveIntervalsAnalysis>();
+ PA.preserve<LiveIntervalsAnalysis>();
return PA;
}
@@ -148,7 +148,7 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
bool VAddrIsZero = false;
if (OldVAddrReg.isVirtual()) {
- if (auto *Def = MRI->getVRegDef(OldVAddrReg))
+ if (auto *Def = MRI->getUniqueVRegDef(OldVAddrReg))
VAddrIsZero = Def->isImplicitDef() ||
(Def->isMoveImmediate() && Def->getOperand(1).isImm() &&
Def->getOperand(1).getImm() == 0);
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index cf127b0bc0d3b..90717395191b6 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -127,6 +127,7 @@
; GCN-O0-NEXT: Fast Register Allocator
; GCN-O0-NEXT: SI Lower WWM Copies
; GCN-O0-NEXT: AMDGPU Reserve WWM Registers
+; GCN-O0-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O0-NEXT: Fast Register Allocator
; GCN-O0-NEXT: SI Fix VGPR copies
; GCN-O0-NEXT: Remove Redundant DEBUG_VALUE analysis
@@ -386,6 +387,7 @@
; GCN-O1-NEXT: SI Lower WWM Copies
; GCN-O1-NEXT: Virtual Register Rewriter
; GCN-O1-NEXT: AMDGPU Reserve WWM Registers
+; GCN-O1-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-NEXT: Virtual Register Map
; GCN-O1-NEXT: Live Register Matrix
; GCN-O1-NEXT: Greedy Register Allocator
@@ -703,6 +705,7 @@
; GCN-O1-OPTS-NEXT: SI Lower WWM Copies
; GCN-O1-OPTS-NEXT: Virtual Register Rewriter
; GCN-O1-OPTS-NEXT: AMDGPU Reserve WWM Registers
+; GCN-O1-OPTS-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-OPTS-NEXT: Virtual Register Map
; GCN-O1-OPTS-NEXT: Live Register Matrix
; GCN-O1-OPTS-NEXT: Greedy Register Allocator
@@ -1025,6 +1028,7 @@
; GCN-O2-NEXT: SI Lower WWM Copies
; GCN-O2-NEXT: Virtual Register Rewriter
; GCN-O2-NEXT: AMDGPU Reserve WWM Registers
+; GCN-O2-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O2-NEXT: Virtual Register Map
; GCN-O2-NEXT: Live Register Matrix
; GCN-O2-NEXT: Greedy Register Allocator
@@ -1360,6 +1364,7 @@
; GCN-O3-NEXT: SI Lower WWM Copies
; GCN-O3-NEXT: Virtual Register Rewriter
; GCN-O3-NEXT: AMDGPU Reserve WWM Registers
+; GCN-O3-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O3-NEXT: Virtual Register Map
; GCN-O3-NEXT: Live Register Matrix
; GCN-O3-NEXT: Greedy Register Allocator
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index a66b2934b086a..fdb7462ffded2 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -11,17 +11,21 @@ define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %
; GFX1250-LABEL: promote_async_load_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr0_sgpr1 killed $exec
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
-; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_ashrrev_i32 v3, 31, v2 :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s1
+; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xfffffe00, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v[2:3], off offset:512
@@ -104,17 +108,21 @@ define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1)
; GFX1250-LABEL: promote_async_store_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr0_sgpr1 killed $exec
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
-; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v1, off
+; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_ashrrev_i32 v3, 31, v2 :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s1
+; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
+; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v1, off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xfffffe00, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[2:3]
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v0, off offset:512
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index c83af33659dad..8ce3d8fc23a45 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -27,6 +27,7 @@
; DEFAULT-NEXT: SI Lower WWM Copies
; DEFAULT-NEXT: Virtual Register Rewriter
; DEFAULT-NEXT: AMDGPU Reserve WWM Registers
+; DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-NEXT: Virtual Register Map
; DEFAULT-NEXT: Live Register Matrix
; DEFAULT-NEXT: Greedy Register Allocator
@@ -46,6 +47,7 @@
; O0-NEXT: Fast Register Allocator
; O0-NEXT: SI Lower WWM Copies
; O0-NEXT: AMDGPU Reserve WWM Registers
+; O0-NEXT: SI Fix Xcnt Stall SAddr Reuse
; O0-NEXT: Fast Register Allocator
; O0-NEXT: SI Fix VGPR copies
@@ -74,6 +76,7 @@
; BASIC-DEFAULT-NEXT: SI Lower WWM Copies
; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
; BASIC-DEFAULT-NEXT: AMDGPU Reserve WWM Registers
+; BASIC-DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-DEFAULT-NEXT: Virtual Register Map
; BASIC-DEFAULT-NEXT: Live Register Matrix
; BASIC-DEFAULT-NEXT: Greedy Register Allocator
@@ -97,6 +100,7 @@
; DEFAULT-BASIC-NEXT: SI Lower WWM Copies
; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
; DEFAULT-BASIC-NEXT: AMDGPU Reserve WWM Registers
+; DEFAULT-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-BASIC-NEXT: Virtual Register Map
; DEFAULT-BASIC-NEXT: Live Register Matrix
; DEFAULT-BASIC-NEXT: Basic Register Allocator
@@ -126,6 +130,7 @@
; BASIC-BASIC-NEXT: SI Lower WWM Copies
; BASIC-BASIC-NEXT: Virtual Register Rewriter
; BASIC-BASIC-NEXT: AMDGPU Reserve WWM Registers
+; BASIC-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-BASIC-NEXT: Virtual Register Map
; BASIC-BASIC-NEXT: Live Register Matrix
; BASIC-BASIC-NEXT: Basic Register Allocator
>From a27dcbb99c6a34a67b11e293f82c8d4b6c2abdc2 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Tue, 7 Apr 2026 11:00:31 -0500
Subject: [PATCH 13/16] review changes
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 4 -
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 36 ++++----
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 1 -
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 1 -
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 1 -
.../wait-xcnt-atomic-rmw-optimization.ll | 87 +++++++++----------
6 files changed, 58 insertions(+), 72 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b8ecb59f7ece7..d0658e12e159b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1825,8 +1825,6 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
addPass(&SILowerWWMCopiesLegacyID);
addPass(&AMDGPUReserveWWMRegsLegacyID);
- addPass(&SIFixXcntStallSAddrReuseLegacyID);
-
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(false));
@@ -2474,8 +2472,6 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
addMachineFunctionPass(SILowerWWMCopiesPass(), PMW);
addMachineFunctionPass(AMDGPUReserveWWMRegsPass(), PMW);
- addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
-
// VGPR allocation - default to fast at -O0.
if (VGPRRegAllocNPM == RegAllocType::Greedy)
addMachineFunctionPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index 450c9bf7749b9..2d1637a2fcb17 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -39,7 +39,7 @@ class SIFixXcntStallSAddrReuse {
MachineRegisterInfo *MRI = nullptr;
LiveIntervals *LIS = nullptr;
- bool sAddrRedefinedAfter(MachineInstr &MI, Register SAddr);
+ bool isSAddrRedefined(MachineInstr &MI, Register SAddr);
MachineInstr *convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
bool processMBB(MachineBasicBlock &MBB);
@@ -55,10 +55,12 @@ class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
SIFixXcntStallSAddrReuseLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override {
+ if (skipFunction(MF.getFunction()))
+ return false;
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return false;
- auto *LISWrapper = getAnalysisIfAvailable<llvm::LiveIntervalsWrapperPass>();
- llvm::LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
+ auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
+ LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
return SIFixXcntStallSAddrReuse(LIS).run(MF);
}
@@ -100,25 +102,21 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
const SIRegisterInfo &TRI,
const MachineRegisterInfo &MRI, int &SAddrIdx) {
+ if (!TII.isFLATGlobal(MI))
+ return false;
SAddrIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::saddr);
- if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ if (SAddrIdx < 0)
return false;
- const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
- return SAddr.isReg() && SAddr.getReg().isPhysical() &&
- TRI.isSGPRReg(MRI, SAddr.getReg());
+ Register SAddr = MI.getOperand(SAddrIdx).getReg();
+ return SAddr.isPhysical() && TRI.isSGPRReg(MRI, SAddr);
}
-bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
- Register SAddr) {
- MachineBasicBlock::iterator I = std::next(MI.getIterator());
- MachineBasicBlock::iterator E = MI.getParent()->end();
- for (; I != E; ++I) {
- for (const MachineOperand &MO : I->operands())
- if (MO.isReg() && MO.isDef() && MO.getReg().isPhysical() &&
- TRI->regsOverlap(MO.getReg(), SAddr))
- return true;
- }
- return false;
+bool SIFixXcntStallSAddrReuse::isSAddrRedefined(MachineInstr &MI,
+ Register SAddr) {
+ return llvm::any_of(
+ instructionsWithoutDebug(std::next(MachineBasicBlock::iterator(MI)),
+ MI.getParent()->end()),
+ [&](const MachineInstr &I) { return I.modifiesRegister(SAddr, TRI); });
}
MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
@@ -253,7 +251,7 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
int SAddrIdx;
if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
continue;
- if (!sAddrRedefinedAfter(MI, MI.getOperand(SAddrIdx).getReg()))
+ if (!isSAddrRedefined(MI, MI.getOperand(SAddrIdx).getReg()))
continue;
Register NewAddr;
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index b6453efe2085f..ca7e9dde9067c 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -68,7 +68,6 @@
; GCN-O0-NEXT: regallocfast<filter=wwm;no-clear-vregs>
; GCN-O0-NEXT: si-lower-wwm-copies
; GCN-O0-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O0-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O0-NEXT: regallocfast<filter=vgpr>
; GCN-O0-NEXT: si-fix-vgpr-copies
; GCN-O0-NEXT: remove-redundant-debug-values
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 90717395191b6..4dc73cde7f358 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -127,7 +127,6 @@
; GCN-O0-NEXT: Fast Register Allocator
; GCN-O0-NEXT: SI Lower WWM Copies
; GCN-O0-NEXT: AMDGPU Reserve WWM Registers
-; GCN-O0-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O0-NEXT: Fast Register Allocator
; GCN-O0-NEXT: SI Fix VGPR copies
; GCN-O0-NEXT: Remove Redundant DEBUG_VALUE analysis
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 8ce3d8fc23a45..18f570f3efd48 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -47,7 +47,6 @@
; O0-NEXT: Fast Register Allocator
; O0-NEXT: SI Lower WWM Copies
; O0-NEXT: AMDGPU Reserve WWM Registers
-; O0-NEXT: SI Fix Xcnt Stall SAddr Reuse
; O0-NEXT: Fast Register Allocator
; O0-NEXT: SI Fix VGPR copies
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 9f07d54a0f5dc..04fad1cb91649 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -1045,49 +1045,47 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
-; GFX1250-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_writelane_b32 v3, s2, 0
-; GFX1250-NEXT: v_writelane_b32 v3, s3, 1
+; GFX1250-NEXT: v_writelane_b32 v2, s2, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s3, 1
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v3, s1, 2
-; GFX1250-NEXT: v_writelane_b32 v3, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v2, s1, 2
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: .LBB17_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v3, 3
-; GFX1250-NEXT: v_readlane_b32 s1, v3, 2
-; GFX1250-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1250-NEXT: v_readlane_b32 s3, v3, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 3
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 2
+; GFX1250-NEXT: v_readlane_b32 s2, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s3, v2, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v2, s0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, s0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1098,9 +1096,9 @@ define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-NEXT: s_mov_b32 s2, -1
; GFX1250-NEXT: s_xor_b32 s1, s1, s2
; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s1
-; GFX1250-NEXT: v_writelane_b32 v3, s0, 3
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 3
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_vccnz .LBB17_1
@@ -1135,62 +1133,60 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX1250-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane
-; GFX1250-NEXT: v_writelane_b32 v3, s4, 0
-; GFX1250-NEXT: v_writelane_b32 v3, s5, 1
+; GFX1250-NEXT: ; implicit-def: $vgpr2 : SGPR spill to VGPR lane
+; GFX1250-NEXT: v_writelane_b32 v2, s4, 0
+; GFX1250-NEXT: v_writelane_b32 v2, s5, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 1
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 2
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: v_mov_b32_e32 v1, 2
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:4 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, -1
; GFX1250-NEXT: s_mov_b32 s2, 0
; GFX1250-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1250-NEXT: v_writelane_b32 v3, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
; GFX1250-NEXT: s_mov_b32 s6, exec_lo
; GFX1250-NEXT: s_mov_b32 exec_lo, -1
-; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_cbranch_scc1 .LBB18_3
; GFX1250-NEXT: .LBB18_1: ; %Flow
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v3, 2
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 2
; GFX1250-NEXT: s_mov_b32 s1, -1
; GFX1250-NEXT: s_xor_b32 s0, s0, s1
; GFX1250-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_vccnz .LBB18_4
; GFX1250-NEXT: ; %bb.2: ; %bb1
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 3
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1206,40 +1202,39 @@ define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %co
; GFX1250-NEXT: s_branch .LBB18_4
; GFX1250-NEXT: .LBB18_3: ; %bb2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 4
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-NEXT: v_mov_b32_e32 v1, 4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_atomic_add_u32 v[0:1], v2, off offset:12 scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:12 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, 0
-; GFX1250-NEXT: v_writelane_b32 v3, s0, 2
+; GFX1250-NEXT: v_writelane_b32 v2, s0, 2
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_store_b32 off, v3, off nv ; 4-byte Folded Spill
+; GFX1250-NEXT: scratch_store_b32 off, v2, off nv ; 4-byte Folded Spill
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_branch .LBB18_1
; GFX1250-NEXT: .LBB18_4: ; %merge
; GFX1250-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1250-NEXT: scratch_load_b32 v3, off, off nv ; 4-byte Folded Reload
+; GFX1250-NEXT: scratch_load_b32 v2, off, off nv ; 4-byte Folded Reload
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 exec_lo, s6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readlane_b32 s0, v3, 0
-; GFX1250-NEXT: v_readlane_b32 s1, v3, 1
+; GFX1250-NEXT: v_readlane_b32 s0, v2, 0
+; GFX1250-NEXT: v_readlane_b32 s1, v2, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, 5
; GFX1250-NEXT: s_wait_loadcnt 0x0
>From f2f7d6a33c5aa773b318e61bffe893a7e0bcc747 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Tue, 7 Apr 2026 18:00:12 -0500
Subject: [PATCH 14/16] moved before virtregrewrtier
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 +-
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 103 +++++++++---
.../AMDGPU/insert_vector_elt.v2bf16.ll | 41 ++---
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 8 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 159 +++++++-----------
.../test/CodeGen/AMDGPU/loop-prefetch-data.ll | 15 +-
.../promote-constOffset-to-imm-gfx12.ll | 16 +-
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 70 +++++---
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 8 +-
10 files changed, 219 insertions(+), 213 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d0658e12e159b..adc7dbfcd77bd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1839,6 +1839,8 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
addPass(createSGPRAllocPass(true));
+ addPass(&SIFixXcntStallSAddrReuseLegacyID);
+
// Commit allocated register changes. This is mostly necessary because too
// many things rely on the use lists of the physical registers, such as the
// verifier. This is only necessary with allocators which use LiveIntervals,
@@ -1862,8 +1864,6 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
addPass(createVirtRegRewriter(false));
addPass(&AMDGPUReserveWWMRegsLegacyID);
- addPass(&SIFixXcntStallSAddrReuseLegacyID);
-
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(true));
@@ -2539,6 +2539,8 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
else
addMachineFunctionPass(RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
+ addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
+
// Commit allocated register changes. This is mostly necessary because too
// many things rely on the use lists of the physical registers, such as the
// verifier. This is only necessary with allocators which use LiveIntervals,
@@ -2566,8 +2568,6 @@ Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
addMachineFunctionPass(VirtRegRewriterPass(false), PMW);
addMachineFunctionPass(AMDGPUReserveWWMRegsPass(), PMW);
- addMachineFunctionPass(SIFixXcntStallSAddrReusePass(), PMW);
-
// VGPR allocation - default to greedy at -O1 and above.
if (VGPRRegAllocNPM == RegAllocType::Fast)
addMachineFunctionPass(RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index 2d1637a2fcb17..6fe7a35a8119e 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -6,11 +6,12 @@
//
//===----------------------------------------------------------------------===//
/// \file
-/// Convert FLAT global SADDR memory ops to VADDR when the physical SGPR
-/// address pair is redefined later in the block. This avoids s_wait_xcnt
-/// stalls from XNACK replay hazards on reused SGPR pairs.
+/// Convert FLAT global SADDR memory ops to VADDR when the SGPR address pair
+/// would be reused across multiple loads. Runs after SGPR RA (greedy) but
+/// before VirtRegRewriter, so saddr operands are still virtual registers
+/// with physical assignments available in VirtRegMap.
///
-/// Runs after SGPR RA and before VGPR RA.
+/// This avoids s_wait_xcnt stalls from XNACK replay hazards on reused.
//===----------------------------------------------------------------------===//
#include "SIFixXcntStallSAddrReuse.h"
@@ -19,16 +20,23 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
using namespace llvm;
#define DEBUG_TYPE "si-fix-xcnt-stall-saddr-reuse"
+static cl::opt<unsigned> SAddrReuseWindow(
+ "amdgpu-saddr-reuse-window", cl::init(30), cl::Hidden,
+ cl::desc("Number of instructions to search for SAddr redefinition"));
+
STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
namespace {
@@ -38,14 +46,18 @@ class SIFixXcntStallSAddrReuse {
const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr;
LiveIntervals *LIS = nullptr;
+ VirtRegMap *VRM = nullptr;
+ MCRegister getPhysForSAddr(Register Reg) const;
bool isSAddrRedefined(MachineInstr &MI, Register SAddr);
MachineInstr *convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
bool processMBB(MachineBasicBlock &MBB);
public:
- SIFixXcntStallSAddrReuse(LiveIntervals *LIS = nullptr) : LIS(LIS) {}
+ SIFixXcntStallSAddrReuse(LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr)
+ : LIS(LIS), VRM(VRM) {}
bool run(MachineFunction &MF);
};
@@ -61,7 +73,9 @@ class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
return false;
auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
- return SIFixXcntStallSAddrReuse(LIS).run(MF);
+ auto *VRMWrapper = getAnalysisIfAvailable<VirtRegMapWrapperLegacy>();
+ VirtRegMap *VRM = VRMWrapper ? &VRMWrapper->getVRM() : nullptr;
+ return SIFixXcntStallSAddrReuse(LIS, VRM).run(MF);
}
StringRef getPassName() const override {
@@ -70,6 +84,8 @@ class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
+ AU.addUsedIfAvailable<VirtRegMapWrapperLegacy>();
+ AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
@@ -90,7 +106,8 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
return PreservedAnalyses::all();
auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
- if (!SIFixXcntStallSAddrReuse(LIS).run(MF))
+ auto *VRM = MFAM.getCachedResult<VirtRegMapAnalysis>(MF);
+ if (!SIFixXcntStallSAddrReuse(LIS, VRM).run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
@@ -108,15 +125,40 @@ static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
if (SAddrIdx < 0)
return false;
Register SAddr = MI.getOperand(SAddrIdx).getReg();
- return SAddr.isPhysical() && TRI.isSGPRReg(MRI, SAddr);
+ if (!SAddr.isVirtual() || !TRI.isSGPRReg(MRI, SAddr))
+ return false;
+ unsigned Size = TRI.getRegSizeInBits(*MRI.getRegClass(SAddr));
+ return Size == 64;
+}
+
+MCRegister SIFixXcntStallSAddrReuse::getPhysForSAddr(Register Reg) const {
+ if (!VRM || !Reg.isVirtual())
+ return MCRegister();
+ return VRM->getPhys(Reg);
}
bool SIFixXcntStallSAddrReuse::isSAddrRedefined(MachineInstr &MI,
Register SAddr) {
- return llvm::any_of(
- instructionsWithoutDebug(std::next(MachineBasicBlock::iterator(MI)),
- MI.getParent()->end()),
- [&](const MachineInstr &I) { return I.modifiesRegister(SAddr, TRI); });
+ MCRegister Phys = getPhysForSAddr(SAddr);
+ if (!Phys)
+ return false;
+
+ unsigned Count = 0;
+ for (const MachineInstr &I : instructionsWithoutDebug(
+ std::next(MachineBasicBlock::iterator(MI)), MI.getParent()->end())) {
+ if (Count++ >= SAddrReuseWindow)
+ return false;
+ if (I.modifiesRegister(Phys, TRI))
+ return true;
+ for (const MachineOperand &MO : I.all_defs()) {
+ if (!MO.getReg().isVirtual())
+ continue;
+ MCRegister DefPhys = getPhysForSAddr(MO.getReg());
+ if (DefPhys && TRI->regsOverlap(Phys, DefPhys))
+ return true;
+ }
+ }
+ return false;
}
MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
@@ -160,9 +202,7 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
if (LIS)
LIS->InsertMachineInstrInMaps(*Copy);
} else {
- MCRegister Lo = TRI->getSubReg(SAddrReg, AMDGPU::sub0);
- MCRegister Hi = TRI->getSubReg(SAddrReg, AMDGPU::sub1);
-
+ // new_vaddr = saddr + sext(old_vaddr_32)
Register TmpVAddr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
auto CopyVAddr =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), TmpVAddr).add(VAddr);
@@ -180,8 +220,19 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
Register LoV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register HiV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- auto CopyLo = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV).addReg(Lo);
- auto CopyHi = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV).addReg(Hi);
+ // Copy full 64-bit SGPR to VGPR64 first to avoid subreg issues
+ // with VirtRegRewriter (sub-ranges might not exist).
+ Register SAddrV = MRI->createVirtualRegister(TRI->getVGPR64Class());
+ auto CopyFull =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrV).addReg(SAddrReg);
+ if (LIS)
+ LIS->InsertMachineInstrInMaps(*CopyFull);
+ MachineInstrBuilder CopyLo =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV)
+ .addReg(SAddrV, {}, AMDGPU::sub0);
+ MachineInstrBuilder CopyHi =
+ BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV)
+ .addReg(SAddrV, {}, AMDGPU::sub1);
if (LIS) {
LIS->InsertMachineInstrInMaps(*CopyLo);
LIS->InsertMachineInstrInMaps(*CopyHi);
@@ -211,6 +262,7 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
LIS->InsertMachineInstrInMaps(*AddHi);
if (LIS) {
+ LIS->createAndComputeVirtRegInterval(SAddrV);
LIS->createAndComputeVirtRegInterval(HiExt);
LIS->createAndComputeVirtRegInterval(LoV);
LIS->createAndComputeVirtRegInterval(HiV);
@@ -224,18 +276,15 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
if (i == (unsigned)OldSAddrIdx)
continue;
- if (i == (unsigned)OldVAddrIdx) {
+ if (i == (unsigned)OldVAddrIdx)
MIB.addReg(NewVAddr);
- } else {
+ else
MIB.add(MI.getOperand(i));
- }
}
-
MIB.cloneMemRefs(MI);
- if (LIS) {
+ if (LIS)
LIS->ReplaceMachineInstrInMaps(MI, *MIB);
- }
LLVM_DEBUG(dbgs() << " Converted: " << *MIB);
MI.eraseFromParent();
@@ -255,8 +304,11 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
continue;
Register NewAddr;
- if (convertToVAddr(MI, MBB, NewAddr))
+ if (convertToVAddr(MI, MBB, NewAddr)) {
+ if (VRM)
+ VRM->grow();
Changed = true;
+ }
}
return Changed;
@@ -268,6 +320,9 @@ bool SIFixXcntStallSAddrReuse::run(MachineFunction &MF) {
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
+ if (VRM)
+ VRM->grow();
+
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
Changed |= processMBB(MBB);
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 70de730d409fc..3d91e82f1b357 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -2155,27 +2155,16 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
-; GFX1250-FAKE16-NEXT: ; kill: killed $sgpr2_sgpr3
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
-; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0 killed $vgpr8 killed $exec
-; GFX1250-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-FAKE16-NEXT: ; kill: def $vgpr4 killed $vgpr8 killed $exec
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_mov_b32 v4, s2
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-FAKE16-NEXT: s_clause 0x1
+; GFX1250-FAKE16-NEXT: global_load_b128 v[0:3], v8, s[2:3]
+; GFX1250-FAKE16-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 6
+; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
-; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
-; GFX1250-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, v4, v4
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 7
-; GFX1250-FAKE16-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v6, s3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v6, v5, vcc_lo
-; GFX1250-FAKE16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v9, v3, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
@@ -2248,26 +2237,16 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv
-; GFX1250-REAL16-NEXT: ; kill: killed $sgpr2_sgpr3
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-REAL16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr8 killed $exec
-; GFX1250-REAL16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr4 killed $vgpr8 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_mov_b32 v4, s2
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v6, s3
+; GFX1250-REAL16-NEXT: s_clause 0x1
+; GFX1250-REAL16-NEXT: global_load_b128 v[0:3], v8, s[2:3]
+; GFX1250-REAL16-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 6
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-REAL16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
-; GFX1250-REAL16-NEXT: v_add_co_u32 v4, vcc_lo, v4, v4
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v6, v5, vcc_lo
-; GFX1250-REAL16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 7
-; GFX1250-REAL16-NEXT: global_load_b128 v[4:7], v[4:5], off offset:16
; GFX1250-REAL16-NEXT: s_cselect_b32 s3, -1, 0
; GFX1250-REAL16-NEXT: s_cmp_eq_u32 s5, 4
; GFX1250-REAL16-NEXT: s_cselect_b32 s6, -1, 0
@@ -2299,8 +2278,6 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_cselect_b32 s5, -1, 0
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x1
; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.l, v3.l, s4, s2
-; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.h, v3.h, s4, s3
-; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.l, v2.l, s4, s6
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_cndmask_b16 v7.l, v7.l, s4, s12
; GFX1250-REAL16-NEXT: v_cndmask_b16 v7.h, v7.h, s4, s13
@@ -2310,6 +2287,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_cndmask_b16 v5.h, v5.h, s4, s17
; GFX1250-REAL16-NEXT: v_cndmask_b16 v4.l, v4.l, s4, s18
; GFX1250-REAL16-NEXT: v_cndmask_b16 v4.h, v4.h, s4, s5
+; GFX1250-REAL16-NEXT: v_cndmask_b16 v3.h, v3.h, s4, s3
+; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.l, v2.l, s4, s6
; GFX1250-REAL16-NEXT: v_cndmask_b16 v2.h, v2.h, s4, s7
; GFX1250-REAL16-NEXT: v_cndmask_b16 v1.l, v1.l, s4, s8
; GFX1250-REAL16-NEXT: v_cndmask_b16 v1.h, v1.h, s4, s9
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index ca7e9dde9067c..696e0bdc04b3c 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -208,6 +208,7 @@
; GCN-O2-NEXT: si-form-memory-clauses
; GCN-O2-NEXT: amdgpu-pre-ra-long-branch-reg
; GCN-O2-NEXT: greedy<sgpr>
+; GCN-O2-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O2-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O2-NEXT: stack-slot-coloring
; GCN-O2-NEXT: si-lower-sgpr-spills
@@ -216,7 +217,6 @@
; GCN-O2-NEXT: si-lower-wwm-copies
; GCN-O2-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O2-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O2-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O2-NEXT: greedy<vgpr>
; GCN-O2-NEXT: amdgpu-nsa-reassign
; GCN-O2-NEXT: amdgpu-rewrite-agpr-copy-mfma
@@ -378,6 +378,7 @@
; GCN-O3-NEXT: si-form-memory-clauses
; GCN-O3-NEXT: amdgpu-pre-ra-long-branch-reg
; GCN-O3-NEXT: greedy<sgpr>
+; GCN-O3-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O3-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O3-NEXT: stack-slot-coloring
; GCN-O3-NEXT: si-lower-sgpr-spills
@@ -386,7 +387,6 @@
; GCN-O3-NEXT: si-lower-wwm-copies
; GCN-O3-NEXT: virt-reg-rewriter<no-clear-vregs>
; GCN-O3-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O3-NEXT: si-fix-xcnt-stall-saddr-reuse
; GCN-O3-NEXT: greedy<vgpr>
; GCN-O3-NEXT: amdgpu-nsa-reassign
; GCN-O3-NEXT: amdgpu-rewrite-agpr-copy-mfma
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 4dc73cde7f358..dfc020ede6e64 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -375,6 +375,7 @@
; GCN-O1-NEXT: Lazy Machine Block Frequency Analysis
; GCN-O1-NEXT: Machine Optimization Remark Emitter
; GCN-O1-NEXT: Greedy Register Allocator
+; GCN-O1-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-NEXT: Virtual Register Rewriter
; GCN-O1-NEXT: Stack Slot Coloring
; GCN-O1-NEXT: SI lower SGPR spill instructions
@@ -386,7 +387,6 @@
; GCN-O1-NEXT: SI Lower WWM Copies
; GCN-O1-NEXT: Virtual Register Rewriter
; GCN-O1-NEXT: AMDGPU Reserve WWM Registers
-; GCN-O1-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-NEXT: Virtual Register Map
; GCN-O1-NEXT: Live Register Matrix
; GCN-O1-NEXT: Greedy Register Allocator
@@ -693,6 +693,7 @@
; GCN-O1-OPTS-NEXT: Lazy Machine Block Frequency Analysis
; GCN-O1-OPTS-NEXT: Machine Optimization Remark Emitter
; GCN-O1-OPTS-NEXT: Greedy Register Allocator
+; GCN-O1-OPTS-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-OPTS-NEXT: Virtual Register Rewriter
; GCN-O1-OPTS-NEXT: Stack Slot Coloring
; GCN-O1-OPTS-NEXT: SI lower SGPR spill instructions
@@ -704,7 +705,6 @@
; GCN-O1-OPTS-NEXT: SI Lower WWM Copies
; GCN-O1-OPTS-NEXT: Virtual Register Rewriter
; GCN-O1-OPTS-NEXT: AMDGPU Reserve WWM Registers
-; GCN-O1-OPTS-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O1-OPTS-NEXT: Virtual Register Map
; GCN-O1-OPTS-NEXT: Live Register Matrix
; GCN-O1-OPTS-NEXT: Greedy Register Allocator
@@ -1016,6 +1016,7 @@
; GCN-O2-NEXT: Lazy Machine Block Frequency Analysis
; GCN-O2-NEXT: Machine Optimization Remark Emitter
; GCN-O2-NEXT: Greedy Register Allocator
+; GCN-O2-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O2-NEXT: Virtual Register Rewriter
; GCN-O2-NEXT: Stack Slot Coloring
; GCN-O2-NEXT: SI lower SGPR spill instructions
@@ -1027,7 +1028,6 @@
; GCN-O2-NEXT: SI Lower WWM Copies
; GCN-O2-NEXT: Virtual Register Rewriter
; GCN-O2-NEXT: AMDGPU Reserve WWM Registers
-; GCN-O2-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O2-NEXT: Virtual Register Map
; GCN-O2-NEXT: Live Register Matrix
; GCN-O2-NEXT: Greedy Register Allocator
@@ -1352,6 +1352,7 @@
; GCN-O3-NEXT: Lazy Machine Block Frequency Analysis
; GCN-O3-NEXT: Machine Optimization Remark Emitter
; GCN-O3-NEXT: Greedy Register Allocator
+; GCN-O3-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O3-NEXT: Virtual Register Rewriter
; GCN-O3-NEXT: Stack Slot Coloring
; GCN-O3-NEXT: SI lower SGPR spill instructions
@@ -1363,7 +1364,6 @@
; GCN-O3-NEXT: SI Lower WWM Copies
; GCN-O3-NEXT: Virtual Register Rewriter
; GCN-O3-NEXT: AMDGPU Reserve WWM Registers
-; GCN-O3-NEXT: SI Fix Xcnt Stall SAddr Reuse
; GCN-O3-NEXT: Virtual Register Map
; GCN-O3-NEXT: Live Register Matrix
; GCN-O3-NEXT: Greedy Register Allocator
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 14b22b924b689..4b9fdf3a768dc 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -1272,18 +1272,18 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v3, s[2:3] nv
+; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 2
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: global_store_b96 v3, v[0:2], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <3 x i1>, ptr addrspace(4) %in
@@ -1488,19 +1488,18 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v4, s[2:3] nv
+; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_and_b32 s4, s2, 1
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10002
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s4
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s5
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
; GFX1250-REAL16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <4 x i1>, ptr addrspace(4) %in
@@ -1736,13 +1735,12 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-FAKE16-NEXT: global_load_u8 v0, v[0:1], off nv
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: global_load_u8 v0, v8, s[2:3] nv
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX1250-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10005
@@ -1763,10 +1761,10 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[0:1], off nv
-; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v8, s[2:3] nv
+; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s5, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
@@ -1777,11 +1775,10 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10004
; GFX1250-REAL16-NEXT: s_lshr_b32 s9, s5, 7
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s5, 0x10006
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s2
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v4, s7
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s9
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v6, s8
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v7, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v2, s5
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v5, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s3
; GFX1250-REAL16-NEXT: s_clause 0x1
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
@@ -2136,10 +2133,10 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: global_load_u16 v0, v[0:1], off nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v0, v16, s[2:3] nv
+; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
@@ -2158,15 +2155,14 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
-; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
-; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
-; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
-; GFX1250-NEXT: v_mov_b32_e32 v14, s14
+; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
+; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v15, s3
+; GFX1250-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s13
+; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v6, s10
+; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v7, s12
+; GFX1250-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v9, s11
+; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v12, s9
+; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v14, s14
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -5488,16 +5484,8 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -5507,6 +5495,7 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
@@ -5740,16 +5729,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -5761,7 +5742,7 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s3
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s4
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s3
; GFX1250-REAL16-NEXT: global_store_b64 v1, v[0:1], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
@@ -6022,16 +6003,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-FAKE16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, s3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-FAKE16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
-; GFX1250-FAKE16-NEXT: global_load_u8 v0, v[2:3], off nv
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -6039,7 +6012,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 3, v0
; GFX1250-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s3
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s3
; GFX1250-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -6057,14 +6030,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr1 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v0, 31, v0 :: v_dual_mov_b32 v2, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, s3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v0, vcc_lo
-; GFX1250-REAL16-NEXT: global_load_u8 v0, v[2:3], off nv
+; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
@@ -6072,7 +6039,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
; GFX1250-REAL16-NEXT: s_and_b32 s4, 0xffff, s4
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, v1
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
@@ -6632,13 +6599,13 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: global_load_u8 v0, v[0:1], off nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u8 v0, v16, s[2:3] nv
+; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v10, s3
+; GFX1250-NEXT: v_mov_b32_e32 v10, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 6
; GFX1250-NEXT: s_lshr_b32 s4, s3, 7
; GFX1250-NEXT: s_lshr_b32 s6, s3, 4
@@ -7006,45 +6973,34 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-REAL16-NEXT: ; kill: def $vgpr0 killed $vgpr3 killed $exec
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_mov_b32 v0, s2
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v0
-; GFX1250-REAL16-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo
-; GFX1250-REAL16-NEXT: global_load_u16 v0, v[0:1], off nv
-; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-REAL16-NEXT: global_load_u16 v0, v3, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
; GFX1250-REAL16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000a
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v5, v3
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10009
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s3, 0x10005
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
@@ -7054,11 +7010,12 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s2
@@ -7431,13 +7388,13 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: global_load_u16 v0, v[0:1], off nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v32, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
+; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_mov_b32 v32, 0 :: v_dual_mov_b32 v28, s3
+; GFX1250-NEXT: v_mov_b32_e32 v28, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index 60a658d33e11b..348d0617411e9 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -243,10 +243,10 @@ define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrsp
; GFX1250-NEXT: s_add_co_i32 s6, s6, -1
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 16
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
-; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr0_sgpr1 killed $exec
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
+; GFX1250-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
; GFX1250-NEXT: s_cbranch_scc1 .LBB1_2
; GFX1250-NEXT: .LBB1_3: ; %for.end
; GFX1250-NEXT: s_endpgm
@@ -362,20 +362,19 @@ define amdgpu_kernel void @copy_constant(ptr addrspace(1) nocapture %d, ptr addr
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: .LBB2_2: ; %for.body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: global_prefetch_b8 v[2:3], off offset:176 scope:SCOPE_SE
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_prefetch_b8 v0, s[2:3] offset:176 scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[8:11], s[2:3], 0x0 nv
; GFX1250-NEXT: s_add_co_i32 s6, s6, -1
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 16
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
-; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr0_sgpr1 killed $exec
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
-; GFX1250-NEXT: global_store_b128 v[6:7], v[2:5], off
+; GFX1250-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 16
; GFX1250-NEXT: s_cbranch_scc1 .LBB2_2
; GFX1250-NEXT: .LBB2_3: ; %for.end
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index fdb7462ffded2..daa0dce655942 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -13,15 +13,15 @@ define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
+; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 31, v2
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_ashrrev_i32 v3, 31, v2 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT: v_mov_b32_e32 v4, s1
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
@@ -110,15 +110,15 @@ define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1)
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
+; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 31, v2
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_ashrrev_i32 v3, 31, v2 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT: v_mov_b32_e32 v4, s1
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v3, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v1, off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index d9c234c636ec6..bcfb57be42eb9 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
---
name: global_sadd_load
@@ -13,8 +13,9 @@ body: |
; GFX1250: liveins: $sgpr44_sgpr45
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
@@ -22,11 +23,13 @@ body: |
; GFX942: liveins: $sgpr44_sgpr45
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %2, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
S_ENDPGM 0, implicit %1
...
@@ -43,8 +46,9 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX1250-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[DEF1]], 0, 0, implicit $exec, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY1]], [[DEF1]], 0, 0, implicit $exec, implicit $exec :: (store (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0
;
@@ -53,12 +57,14 @@ body: |
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; GFX942-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], [[COPY]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
; GFX942-NEXT: S_ENDPGM 0
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
- GLOBAL_STORE_DWORD_SADDR %0, %1, $sgpr44_sgpr45, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ GLOBAL_STORE_DWORD_SADDR %0, %1, %2, 0, 0, implicit $exec :: (store (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
S_ENDPGM 0
...
@@ -74,13 +80,15 @@ body: |
; GFX1250: liveins: $sgpr44_sgpr45, $vgpr10
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
- ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY1]], implicit $exec
- ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $sgpr44
- ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $sgpr45
- ; GFX1250-NEXT: undef %2.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY2]], [[COPY1]], 0, implicit $exec
- ; GFX1250-NEXT: %2.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY3]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %2, 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: dead [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY2]], implicit $exec
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[COPY1]]
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY3]].sub0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]].sub1
+ ; GFX1250-NEXT: undef %3.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY4]], [[COPY2]], 0, implicit $exec
+ ; GFX1250-NEXT: dead %3.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY5]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %3, 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
;
@@ -88,11 +96,13 @@ body: |
; GFX942: liveins: $sgpr44_sgpr45, $vgpr10
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
%0:vgpr_32 = COPY $vgpr10
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %2, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
S_ENDPGM 0, implicit %1
...
@@ -108,12 +118,14 @@ body: |
; GFX1250: liveins: $sgpr44_sgpr45
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX1250-NEXT: dead [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[COPY2]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY3]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
;
@@ -121,17 +133,21 @@ body: |
; GFX942: liveins: $sgpr44_sgpr45
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
; GFX942-NEXT: $sgpr45 = S_MOV_B32 0
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY1]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
; GFX942-NEXT: $sgpr44 = S_MOV_B32 1
; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
%0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 0
$sgpr45 = S_MOV_B32 0
- %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR $sgpr44_sgpr45, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %4:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %4, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
$sgpr44 = S_MOV_B32 1
S_ENDPGM 0, implicit %1, implicit %2
...
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 18f570f3efd48..8a6afa56ff64e 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -16,6 +16,7 @@
; REGALLOC: -regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, and -vgpr-regalloc
; DEFAULT: Greedy Register Allocator
+; DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-NEXT: Virtual Register Rewriter
; DEFAULT-NEXT: Stack Slot Coloring
; DEFAULT-NEXT: SI lower SGPR spill instructions
@@ -27,7 +28,6 @@
; DEFAULT-NEXT: SI Lower WWM Copies
; DEFAULT-NEXT: Virtual Register Rewriter
; DEFAULT-NEXT: AMDGPU Reserve WWM Registers
-; DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-NEXT: Virtual Register Map
; DEFAULT-NEXT: Live Register Matrix
; DEFAULT-NEXT: Greedy Register Allocator
@@ -60,6 +60,7 @@
; BASIC-DEFAULT-NEXT: Virtual Register Map
; BASIC-DEFAULT-NEXT: Live Register Matrix
; BASIC-DEFAULT-NEXT: Basic Register Allocator
+; BASIC-DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
; BASIC-DEFAULT-NEXT: Stack Slot Coloring
; BASIC-DEFAULT-NEXT: SI lower SGPR spill instructions
@@ -75,7 +76,6 @@
; BASIC-DEFAULT-NEXT: SI Lower WWM Copies
; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
; BASIC-DEFAULT-NEXT: AMDGPU Reserve WWM Registers
-; BASIC-DEFAULT-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-DEFAULT-NEXT: Virtual Register Map
; BASIC-DEFAULT-NEXT: Live Register Matrix
; BASIC-DEFAULT-NEXT: Greedy Register Allocator
@@ -88,6 +88,7 @@
; DEFAULT-BASIC: Greedy Register Allocator
+; DEFAULT-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
; DEFAULT-BASIC-NEXT: Stack Slot Coloring
; DEFAULT-BASIC-NEXT: SI lower SGPR spill instructions
@@ -99,7 +100,6 @@
; DEFAULT-BASIC-NEXT: SI Lower WWM Copies
; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
; DEFAULT-BASIC-NEXT: AMDGPU Reserve WWM Registers
-; DEFAULT-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; DEFAULT-BASIC-NEXT: Virtual Register Map
; DEFAULT-BASIC-NEXT: Live Register Matrix
; DEFAULT-BASIC-NEXT: Basic Register Allocator
@@ -118,6 +118,7 @@
; BASIC-BASIC-NEXT: Virtual Register Map
; BASIC-BASIC-NEXT: Live Register Matrix
; BASIC-BASIC-NEXT: Basic Register Allocator
+; BASIC-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-BASIC-NEXT: Virtual Register Rewriter
; BASIC-BASIC-NEXT: Stack Slot Coloring
; BASIC-BASIC-NEXT: SI lower SGPR spill instructions
@@ -129,7 +130,6 @@
; BASIC-BASIC-NEXT: SI Lower WWM Copies
; BASIC-BASIC-NEXT: Virtual Register Rewriter
; BASIC-BASIC-NEXT: AMDGPU Reserve WWM Registers
-; BASIC-BASIC-NEXT: SI Fix Xcnt Stall SAddr Reuse
; BASIC-BASIC-NEXT: Virtual Register Map
; BASIC-BASIC-NEXT: Live Register Matrix
; BASIC-BASIC-NEXT: Basic Register Allocator
>From dd0347cc06399a47adec6bf72c6d7a7ece418fc7 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 8 Apr 2026 11:30:59 -0500
Subject: [PATCH 15/16] fix stale LIS update and redo group based kills
---
.../AMDGPU/SIFixXcntStallSAddrReuse.cpp | 184 ++++----
.../AMDGPU/GlobalISel/load-constant.96.ll | 8 +-
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 88 ++--
.../test/CodeGen/AMDGPU/loop-prefetch-data.ll | 10 +-
.../promote-constOffset-to-imm-gfx12.ll | 22 +-
llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir | 434 +++++++++++++-----
6 files changed, 492 insertions(+), 254 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index 6fe7a35a8119e..890e21a050a0f 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -11,13 +11,13 @@
/// before VirtRegRewriter, so saddr operands are still virtual registers
/// with physical assignments available in VirtRegMap.
///
-/// This avoids s_wait_xcnt stalls from XNACK replay hazards on reused.
+/// This avoids s_wait_xcnt stalls from XNACK replay hazards on reused
+/// physical SGPR pairs.
//===----------------------------------------------------------------------===//
#include "SIFixXcntStallSAddrReuse.h"
#include "AMDGPU.h"
#include "GCNSubtarget.h"
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -34,8 +34,14 @@ using namespace llvm;
#define DEBUG_TYPE "si-fix-xcnt-stall-saddr-reuse"
static cl::opt<unsigned> SAddrReuseWindow(
- "amdgpu-saddr-reuse-window", cl::init(30), cl::Hidden,
- cl::desc("Number of instructions to search for SAddr redefinition"));
+ "amdgpu-saddr-reuse-window", cl::init(64), cl::Hidden,
+ cl::desc("Instruction window for SAddr redefinition scan"));
+
+static cl::opt<unsigned> AddrLivenessGroupSize(
+ "amdgpu-saddr-vaddr-liveness-group",
+ cl::desc("Number of memory ops to group and keep their VADDRs "
+ "simultaneously live to prevent immediate reuse"),
+ cl::init(16), cl::Hidden);
STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
@@ -48,7 +54,6 @@ class SIFixXcntStallSAddrReuse {
LiveIntervals *LIS = nullptr;
VirtRegMap *VRM = nullptr;
- MCRegister getPhysForSAddr(Register Reg) const;
bool isSAddrRedefined(MachineInstr &MI, Register SAddr);
MachineInstr *convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
Register &NewAddrReg);
@@ -109,13 +114,21 @@ SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
auto *VRM = MFAM.getCachedResult<VirtRegMapAnalysis>(MF);
if (!SIFixXcntStallSAddrReuse(LIS, VRM).run(MF))
return PreservedAnalyses::all();
- auto PA = getMachineFunctionPassPreservedAnalyses();
+ PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
- if (LIS)
- PA.preserve<LiveIntervalsAnalysis>();
+ PA.preserve<LiveIntervalsAnalysis>();
+ PA.preserve<VirtRegMapAnalysis>();
return PA;
}
+static bool isZeroVAddr(Register Reg, const MachineRegisterInfo &MRI) {
+ if (!Reg.isVirtual())
+ return false;
+ const MachineInstr *Def = MRI.getUniqueVRegDef(Reg);
+ return Def && Def->isMoveImmediate() && Def->getOperand(1).isImm() &&
+ Def->getOperand(1).getImm() == 0;
+}
+
static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
const SIRegisterInfo &TRI,
const MachineRegisterInfo &MRI, int &SAddrIdx) {
@@ -125,21 +138,14 @@ static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
if (SAddrIdx < 0)
return false;
Register SAddr = MI.getOperand(SAddrIdx).getReg();
- if (!SAddr.isVirtual() || !TRI.isSGPRReg(MRI, SAddr))
+ if (!SAddr.isVirtual())
return false;
- unsigned Size = TRI.getRegSizeInBits(*MRI.getRegClass(SAddr));
- return Size == 64;
-}
-
-MCRegister SIFixXcntStallSAddrReuse::getPhysForSAddr(Register Reg) const {
- if (!VRM || !Reg.isVirtual())
- return MCRegister();
- return VRM->getPhys(Reg);
+ return AMDGPU::SReg_64RegClass.hasSubClassEq(MRI.getRegClass(SAddr));
}
bool SIFixXcntStallSAddrReuse::isSAddrRedefined(MachineInstr &MI,
Register SAddr) {
- MCRegister Phys = getPhysForSAddr(SAddr);
+ MCRegister Phys = VRM->getPhys(SAddr);
if (!Phys)
return false;
@@ -148,13 +154,11 @@ bool SIFixXcntStallSAddrReuse::isSAddrRedefined(MachineInstr &MI,
std::next(MachineBasicBlock::iterator(MI)), MI.getParent()->end())) {
if (Count++ >= SAddrReuseWindow)
return false;
- if (I.modifiesRegister(Phys, TRI))
- return true;
for (const MachineOperand &MO : I.all_defs()) {
- if (!MO.getReg().isVirtual())
+ Register DefReg = MO.getReg();
+ if (!DefReg.isVirtual())
continue;
- MCRegister DefPhys = getPhysForSAddr(MO.getReg());
- if (DefPhys && TRI->regsOverlap(Phys, DefPhys))
+ if (TRI->regsOverlap(Phys, VRM->getPhys(DefReg)))
return true;
}
}
@@ -168,7 +172,8 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
if (NewOpc < 0)
return nullptr;
- if (AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr) < 0)
+ int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
+ if (NewVAddrIdx < 0)
return nullptr;
// VADDR form does not support scale_offset.
@@ -186,57 +191,36 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
Register OldVAddrReg = VAddr.getReg();
const DebugLoc &DL = MI.getDebugLoc();
- bool VAddrIsZero = false;
- if (OldVAddrReg.isVirtual()) {
- if (auto *Def = MRI->getUniqueVRegDef(OldVAddrReg))
- VAddrIsZero = Def->isImplicitDef() ||
- (Def->isMoveImmediate() && Def->getOperand(1).isImm() &&
- Def->getOperand(1).getImm() == 0);
- }
+ bool VAddrIsZero = isZeroVAddr(OldVAddrReg, *MRI);
- Register NewVAddr = MRI->createVirtualRegister(TRI->getVGPR64Class());
+ const TargetRegisterClass *VAddrRC =
+ TII->getRegClass(TII->get(NewOpc), NewVAddrIdx);
+ Register NewVAddr = MRI->createVirtualRegister(VAddrRC);
if (VAddrIsZero) {
auto Copy =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), NewVAddr).addReg(SAddrReg);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*Copy);
+ LIS->InsertMachineInstrInMaps(*Copy);
} else {
// new_vaddr = saddr + sext(old_vaddr_32)
Register TmpVAddr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
auto CopyVAddr =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), TmpVAddr).add(VAddr);
- if (LIS) {
- LIS->InsertMachineInstrInMaps(*CopyVAddr);
- LIS->createAndComputeVirtRegInterval(TmpVAddr);
- }
Register HiExt = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
auto Ashr = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ASHRREV_I32_e64), HiExt)
.addImm(31)
.addReg(TmpVAddr);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*Ashr);
-
Register LoV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
Register HiV = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- // Copy full 64-bit SGPR to VGPR64 first to avoid subreg issues
- // with VirtRegRewriter (sub-ranges might not exist).
- Register SAddrV = MRI->createVirtualRegister(TRI->getVGPR64Class());
+ Register SAddrV = MRI->createVirtualRegister(VAddrRC);
auto CopyFull =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SAddrV).addReg(SAddrReg);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*CopyFull);
- MachineInstrBuilder CopyLo =
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV)
- .addReg(SAddrV, {}, AMDGPU::sub0);
- MachineInstrBuilder CopyHi =
- BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV)
- .addReg(SAddrV, {}, AMDGPU::sub1);
- if (LIS) {
- LIS->InsertMachineInstrInMaps(*CopyLo);
- LIS->InsertMachineInstrInMaps(*CopyHi);
- }
+
+ auto CopyLo = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), LoV)
+ .addReg(SAddrV, {}, AMDGPU::sub0);
+ auto CopyHi = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), HiV)
+ .addReg(SAddrV, {}, AMDGPU::sub1);
MCRegister VCC = TRI->getVCC();
auto AddLo =
@@ -247,8 +231,6 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
.addImm(0);
AddLo->getOperand(0).setSubReg(AMDGPU::sub0);
AddLo->getOperand(0).setIsUndef(true);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*AddLo);
auto AddHi =
BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), NewVAddr)
@@ -258,44 +240,75 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
.addReg(VCC, RegState::Kill)
.addImm(0);
AddHi->getOperand(0).setSubReg(AMDGPU::sub1);
- if (LIS)
- LIS->InsertMachineInstrInMaps(*AddHi);
-
- if (LIS) {
- LIS->createAndComputeVirtRegInterval(SAddrV);
- LIS->createAndComputeVirtRegInterval(HiExt);
- LIS->createAndComputeVirtRegInterval(LoV);
- LIS->createAndComputeVirtRegInterval(HiV);
- }
+ LIS->InsertMachineInstrInMaps(*CopyVAddr);
+ LIS->InsertMachineInstrInMaps(*Ashr);
+ LIS->InsertMachineInstrInMaps(*CopyFull);
+ LIS->InsertMachineInstrInMaps(*CopyLo);
+ LIS->InsertMachineInstrInMaps(*CopyHi);
+ LIS->InsertMachineInstrInMaps(*AddLo);
+ LIS->InsertMachineInstrInMaps(*AddHi);
+ LIS->createAndComputeVirtRegInterval(TmpVAddr);
+ LIS->createAndComputeVirtRegInterval(VAddrSignExt);
+ LIS->createAndComputeVirtRegInterval(SAddrV);
+ LIS->createAndComputeVirtRegInterval(SAddrLo);
+ LIS->createAndComputeVirtRegInterval(SAddrHi);
}
- if (LIS)
- LIS->createAndComputeVirtRegInterval(NewVAddr);
-
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(NewOpc));
- for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
+ for (unsigned i = 0, e = MI.getNumOperands(); i < e; ++i) {
+ const MachineOperand &MO = MI.getOperand(i);
+ if (MO.isReg() && MO.isImplicit())
+ continue;
if (i == (unsigned)OldSAddrIdx)
continue;
if (i == (unsigned)OldVAddrIdx)
MIB.addReg(NewVAddr);
else
- MIB.add(MI.getOperand(i));
+ MIB.add(MO);
}
MIB.cloneMemRefs(MI);
- if (LIS)
- LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+ LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+ LIS->createAndComputeVirtRegInterval(NewVAddr);
LLVM_DEBUG(dbgs() << " Converted: " << *MIB);
MI.eraseFromParent();
+
+ if (LIS->hasInterval(SAddrReg))
+ LIS->shrinkToUses(&LIS->getInterval(SAddrReg));
+ if (OldVAddrReg.isVirtual() && LIS->hasInterval(OldVAddrReg))
+ LIS->shrinkToUses(&LIS->getInterval(OldVAddrReg));
+
+ VRM->grow();
+
NewAddrReg = NewVAddr;
++NumConverted;
return MIB.getInstr();
}
bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
+ SmallVector<Register, 8> GroupRegs;
+ MachineInstr *GroupLast = nullptr;
bool Changed = false;
+ auto FlushGroup = [&]() {
+ if (GroupRegs.size() > 1) {
+ auto KillMI =
+ BuildMI(MBB, std::next(GroupLast->getIterator()),
+ GroupLast->getDebugLoc(), TII->get(TargetOpcode::KILL));
+ for (Register R : GroupRegs)
+ KillMI.addReg(R, RegState::Kill);
+ LIS->InsertMachineInstrInMaps(*KillMI);
+ for (Register R : GroupRegs) {
+ if (LIS->hasInterval(R))
+ LIS->removeInterval(R);
+ LIS->createAndComputeVirtRegInterval(R);
+ }
+ }
+ GroupRegs.clear();
+ GroupLast = nullptr;
+ };
+
for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
int SAddrIdx;
if (!isEligible(MI, *TII, *TRI, *MRI, SAddrIdx))
@@ -304,25 +317,30 @@ bool SIFixXcntStallSAddrReuse::processMBB(MachineBasicBlock &MBB) {
continue;
Register NewAddr;
- if (convertToVAddr(MI, MBB, NewAddr)) {
- if (VRM)
- VRM->grow();
- Changed = true;
- }
+ MachineInstr *NewMI = convertToVAddr(MI, MBB, NewAddr);
+ if (!NewMI)
+ continue;
+
+ GroupRegs.push_back(NewAddr);
+ GroupLast = NewMI;
+ Changed = true;
+
+ if (AddrLivenessGroupSize > 0 && GroupRegs.size() >= AddrLivenessGroupSize)
+ FlushGroup();
}
+ FlushGroup();
+
return Changed;
}
bool SIFixXcntStallSAddrReuse::run(MachineFunction &MF) {
+ if (!VRM || !LIS)
+ return false;
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
-
- if (VRM)
- VRM->grow();
-
bool Changed = false;
for (MachineBasicBlock &MBB : MF)
Changed |= processMBB(MBB);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 63dd2d182ae5b..d879c6d787445 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -922,8 +922,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align1(ptr addrspace(4) inreg
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align1:
; GFX1250-UNALIGNED: ; %bb.0:
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-UNALIGNED-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-UNALIGNED-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off nv
; GFX1250-UNALIGNED-NEXT: s_wait_loadcnt 0x0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s0, v0
@@ -1203,8 +1203,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(ptr addrspace(4) inreg
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align2:
; GFX1250-UNALIGNED: ; %bb.0:
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-UNALIGNED-NEXT: ; kill: def $vgpr0_vgpr1 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-UNALIGNED-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off nv
; GFX1250-UNALIGNED-NEXT: s_wait_loadcnt 0x0
; GFX1250-UNALIGNED-NEXT: v_readfirstlane_b32 s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index a981485ac2fad..1a81152f7c1e1 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -102,32 +102,31 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1250-NEXT: s_add_co_i32 s6, s2, 4
; GFX1250-NEXT: s_mov_b32 s7, s2
-; GFX1250-NEXT: v_mov_b32_e32 v2, s6
-; GFX1250-NEXT: s_mov_b32 s6, 2
; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v1, 4
+; GFX1250-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, s6
; GFX1250-NEXT: ; asyncmark
-; GFX1250-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1] offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_mov_b32 s6, 2
+; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; GFX1250-NEXT: ; asyncmark
; GFX1250-NEXT: .LBB1_1: ; %loop_body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_add_co_i32 s8, s7, 8
-; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr0_sgpr1 killed $exec
+; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-NEXT: v_mov_b32_e32 v1, s8
; GFX1250-NEXT: s_add_co_i32 s6, s6, 1
-; GFX1250-NEXT: v_mov_b32_e32 v4, s8
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
-; GFX1250-NEXT: global_load_async_to_lds_b32 v4, v[2:3], off offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, s7
+; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v[2:3], off offset:4 nv
+; GFX1250-NEXT: v_mov_b32_e32 v1, s7
; GFX1250-NEXT: ; asyncmark
; GFX1250-NEXT: ; wait_asyncmark(2)
; GFX1250-NEXT: s_wait_asynccnt 0x2
; GFX1250-NEXT: s_add_co_i32 s7, s7, 4
; GFX1250-NEXT: s_cmp_lt_i32 s6, s3
-; GFX1250-NEXT: ds_load_b32 v2, v2
+; GFX1250-NEXT: ds_load_b32 v1, v1
; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX1250-NEXT: s_cbranch_scc1 .LBB1_1
; GFX1250-NEXT: ; %bb.2: ; %epilog
; GFX1250-NEXT: s_lshl2_add_u32 s0, s3, s2
@@ -135,13 +134,13 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
; GFX1250-NEXT: s_wait_asynccnt 0x1
; GFX1250-NEXT: s_add_co_i32 s0, s0, -8
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
-; GFX1250-NEXT: ds_load_b32 v0, v0
+; GFX1250-NEXT: ds_load_b32 v1, v1
; GFX1250-NEXT: ; wait_asyncmark(0)
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_wait_asynccnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v0
+; GFX1250-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
@@ -208,65 +207,66 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s6, s[8:9], 0x0
; GFX1250-NEXT: s_load_b32 s7, s[0:1], 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s10
+; GFX1250-NEXT: v_mov_b32_e32 v0, s10
; GFX1250-NEXT: s_add_co_i32 s11, s10, 4
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11
+; GFX1250-NEXT: v_dual_mov_b32 v2, 4 :: v_dual_mov_b32 v3, s11
; GFX1250-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
-; GFX1250-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
+; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[8:9] offset:4 nv
; GFX1250-NEXT: ; asyncmark
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
-; GFX1250-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
+; GFX1250-NEXT: global_load_b32 v0, v1, s[8:9] offset:4
+; GFX1250-NEXT: global_load_b32 v1, v1, s[0:1] offset:4
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7
+; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
; GFX1250-NEXT: s_mov_b64 s[6:7], s[2:3]
-; GFX1250-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
+; GFX1250-NEXT: global_load_async_to_lds_b32 v3, v2, s[8:9] offset:4 nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1250-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
; GFX1250-NEXT: s_mov_b32 s8, 2
; GFX1250-NEXT: s_mov_b32 s9, s10
; GFX1250-NEXT: ; asyncmark
; GFX1250-NEXT: .LBB2_1: ; %loop_body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_add_co_i32 s12, s9, 8
+; GFX1250-NEXT: s_wait_xcnt 0x2
+; GFX1250-NEXT: v_mov_b64_e32 v[8:9], s[4:5]
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
+; GFX1250-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
-; GFX1250-NEXT: v_mov_b32_e32 v8, v3
-; GFX1250-NEXT: ; kill: def $vgpr10_vgpr11 killed $sgpr4_sgpr5 killed $exec
-; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v14, s12
+; GFX1250-NEXT: v_mov_b32_e32 v7, v2
+; GFX1250-NEXT: global_load_b32 v2, v[8:9], off
; GFX1250-NEXT: global_load_b32 v3, v[10:11], off
-; GFX1250-NEXT: ; meta instruction
-; GFX1250-NEXT: global_load_b32 v4, v[10:11], off
-; GFX1250-NEXT: ; kill: def $vgpr10_vgpr11 killed $sgpr4_sgpr5 killed $exec
-; GFX1250-NEXT: s_add_co_i32 s8, s8, 1
-; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
-; GFX1250-NEXT: global_load_async_to_lds_b32 v9, v[10:11], off offset:4 nv
-; GFX1250-NEXT: v_mov_b32_e32 v9, s9
+; GFX1250-NEXT: v_dual_add_nc_u32 v15, v4, v5 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: global_load_async_to_lds_b32 v14, v[12:13], off offset:4 nv
+; GFX1250-NEXT: v_mov_b32_e32 v14, s9
; GFX1250-NEXT: ; asyncmark
; GFX1250-NEXT: ; wait_asyncmark(2)
-; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_wait_asynccnt 0x2
; GFX1250-NEXT: s_wait_asynccnt 0x2
-; GFX1250-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
-; GFX1250-NEXT: ds_load_b32 v9, v9
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
+; GFX1250-NEXT: s_add_co_i32 s8, s8, 1
; GFX1250-NEXT: s_add_co_i32 s9, s9, 4
+; GFX1250-NEXT: ds_load_b32 v14, v14
+; GFX1250-NEXT: v_mov_b32_e32 v4, v0
; GFX1250-NEXT: s_cmp_lt_i32 s8, s11
+; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[4:5], 4
; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v9, v10, v9
-; GFX1250-NEXT: ; kill: def $vgpr10_vgpr11 killed $sgpr6_sgpr7 killed $exec
+; GFX1250-NEXT: v_add_nc_u32_e32 v16, v15, v14
+; GFX1250-NEXT: v_mov_b64_e32 v[14:15], s[6:7]
; GFX1250-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4
-; GFX1250-NEXT: global_store_b32 v[10:11], v9, off
+; GFX1250-NEXT: ; kill: killed $vgpr8_vgpr9 killed $vgpr10_vgpr11 killed $vgpr12_vgpr13 killed $vgpr14_vgpr15
+; GFX1250-NEXT: global_store_b32 v[14:15], v16, off
; GFX1250-NEXT: s_cbranch_scc1 .LBB2_1
; GFX1250-NEXT: ; %bb.2: ; %epilog
; GFX1250-NEXT: s_add_co_i32 s0, s11, -2
@@ -274,17 +274,17 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GFX1250-NEXT: s_wait_asynccnt 0x1
; GFX1250-NEXT: s_lshl2_add_u32 s1, s0, s10
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1
+; GFX1250-NEXT: v_dual_add_nc_u32 v4, v7, v6 :: v_dual_mov_b32 v0, s1
; GFX1250-NEXT: ds_load_b32 v1, v0
; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v2, v1
+; GFX1250-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v4, v1
; GFX1250-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
; GFX1250-NEXT: ; wait_asyncmark(0)
; GFX1250-NEXT: s_wait_asynccnt 0x0
; GFX1250-NEXT: ds_load_b32 v0, v0 offset:4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v4
+; GFX1250-NEXT: v_add_nc_u32_e32 v1, v2, v3
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v0
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index 348d0617411e9..b0bee8fd3dc5a 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -236,13 +236,15 @@ define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrsp
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 0xb0
; GFX1250-NEXT: .LBB1_2: ; %for.body
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: ; kill: def $vgpr2_vgpr3 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: ; kill: def $vgpr6_vgpr7 killed $sgpr2_sgpr3 killed $exec
-; GFX1250-NEXT: global_prefetch_b8 v[6:7], off scope:SCOPE_SE
-; GFX1250-NEXT: global_load_b128 v[2:5], v[2:3], off offset:-176
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
+; GFX1250-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
; GFX1250-NEXT: s_add_co_i32 s6, s6, -1
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], 16
; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1250-NEXT: ; kill: killed $vgpr6_vgpr7 killed $vgpr8_vgpr9
+; GFX1250-NEXT: global_load_b128 v[2:5], v[6:7], off offset:-176
+; GFX1250-NEXT: global_prefetch_b8 v[8:9], off scope:SCOPE_SE
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b128 v0, v[2:5], s[0:1]
; GFX1250-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index daa0dce655942..fdfbc93ca5eef 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -13,15 +13,16 @@ define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
-; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
-; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 31, v2
+; GFX1250-NEXT: v_mov_b32_e32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
+; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
@@ -110,15 +111,16 @@ define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1)
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
-; GFX1250-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
-; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 31, v2
+; GFX1250-NEXT: v_mov_b32_e32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v2
+; GFX1250-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo
; GFX1250-NEXT: global_store_async_from_lds_b128 v[2:3], v1, off
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff00
diff --git a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
index bcfb57be42eb9..850cbce0e240f 100644
--- a/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/saddr-to-vaddr.mir
@@ -1,153 +1,369 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -verify-machineinstrs -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -verify-machineinstrs -run-pass=greedy,si-fix-xcnt-stall-saddr-reuse -o - %s | FileCheck -check-prefix=GFX942 %s
+# Two loads using different virtual SGPRs that RA assigns to the same physical
+# pair. The first load's saddr is "redefined" (via VRM) by the second load's
+# saddr definition. Zero vaddr (V_MOV_B32_e32 0) exercises the simple COPY path.
---
-name: global_sadd_load
+name: zero_vaddr_reuse
tracksRegLiveness: true
+registers:
+ - { id: 0, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 4, class: vgpr_32 }
body: |
bb.0:
- liveins: $sgpr44_sgpr45
+ liveins: $sgpr0_sgpr1
- ; GFX1250-LABEL: name: global_sadd_load
- ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-LABEL: name: zero_vaddr_reuse
+ ; GFX1250: liveins: $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
- ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
;
- ; GFX942-LABEL: name: global_sadd_load
- ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-LABEL: name: zero_vaddr_reuse
+ ; GFX942: liveins: $sgpr0_sgpr1
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = IMPLICIT_DEF
- %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %2, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ %0:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %0, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ %4:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_ENDPGM 0, implicit %2, implicit %4
+...
+
+# Same scenario but with a non-zero vaddr offset, exercises the full
+# sext + add path.
+---
+name: nonzero_vaddr_reuse
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 4, class: vgpr_32 }
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1, $vgpr0
+
+ ; GFX1250-LABEL: name: nonzero_vaddr_reuse
+ ; GFX1250: liveins: $sgpr0_sgpr1, $vgpr0
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY1]], implicit $exec
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM]]
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub0
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub1
+ ; GFX1250-NEXT: undef %5.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY3]], [[COPY1]], 0, implicit $exec
+ ; GFX1250-NEXT: %5.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY4]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %5, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
+ ;
+ ; GFX942-LABEL: name: nonzero_vaddr_reuse
+ ; GFX942: liveins: $sgpr0_sgpr1, $vgpr0
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM]], [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ %0:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ %1:vgpr_32 = COPY $vgpr0
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %0, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ %4:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_ENDPGM 0, implicit %2, implicit %4
...
+# Store variant — verifies stores are also converted.
---
-name: global_sadd_store
+name: store_reuse
tracksRegLiveness: true
+registers:
+ - { id: 0, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
body: |
bb.0:
- liveins: $sgpr44_sgpr45
+ liveins: $sgpr0_sgpr1, $vgpr0
- ; GFX1250-LABEL: name: global_sadd_store
- ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-LABEL: name: store_reuse
+ ; GFX1250: liveins: $sgpr0_sgpr1, $vgpr0
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY1]], [[DEF1]], 0, 0, implicit $exec, implicit $exec :: (store (s32), addrspace 1)
- ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM]]
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_]], [[COPY]], [[S_LOAD_DWORDX2_IMM1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
;
- ; GFX942-LABEL: name: global_sadd_store
- ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-LABEL: name: store_reuse
+ ; GFX942: liveins: $sgpr0_sgpr1, $vgpr0
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF]], [[DEF1]], [[COPY]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_]], [[COPY]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_]], [[COPY]], [[S_LOAD_DWORDX2_IMM1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
- %0:vgpr_32 = IMPLICIT_DEF
- %1:vgpr_32 = IMPLICIT_DEF
- %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- GLOBAL_STORE_DWORD_SADDR %0, %1, %2, 0, 0, implicit $exec :: (store (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 0
+ %0:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %2:vgpr_32 = COPY $vgpr0
+ GLOBAL_STORE_DWORD_SADDR %1, %2, %0, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ GLOBAL_STORE_DWORD_SADDR %1, %2, %3, 0, 0, implicit $exec :: (store (s32), addrspace 1)
S_ENDPGM 0
...
+# No reuse — different preferred registers, pass should not convert.
---
-name: nonzero_vaddr
+name: no_reuse
tracksRegLiveness: true
+registers:
+ - { id: 0, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 1, class: vgpr_32 }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sreg_64_xexec_xnull, preferred-register: '$sgpr6_sgpr7' }
+ - { id: 4, class: vgpr_32 }
body: |
bb.0:
- liveins: $sgpr44_sgpr45, $vgpr10
+ liveins: $sgpr0_sgpr1
- ; GFX1250-LABEL: name: nonzero_vaddr
- ; GFX1250: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX1250-LABEL: name: no_reuse
+ ; GFX1250: liveins: $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: dead [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
- ; GFX1250-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[COPY2]], implicit $exec
- ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[COPY1]]
- ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY3]].sub0
- ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY3]].sub1
- ; GFX1250-NEXT: undef %3.sub0:vreg_64_align2, $vcc_lo = V_ADD_CO_U32_e64 [[COPY4]], [[COPY2]], 0, implicit $exec
- ; GFX1250-NEXT: dead %3.sub1:vreg_64_align2, dead $vcc_lo = V_ADDC_U32_e64 [[COPY5]], [[V_ASHRREV_I32_e64_]], killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD %3, 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
- ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]]
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
;
- ; GFX942-LABEL: name: nonzero_vaddr
- ; GFX942: liveins: $sgpr44_sgpr45, $vgpr10
+ ; GFX942-LABEL: name: no_reuse
+ ; GFX942: liveins: $sgpr0_sgpr1
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; GFX942-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]]
- %0:vgpr_32 = COPY $vgpr10
- %2:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %2, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 0
- S_ENDPGM 0, implicit %1
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
+ %0:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %0, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ %4:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %1, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_ENDPGM 0, implicit %2, implicit %4
...
+# 18 loads with the same preferred SGPR pair — triggers FlushGroup/KILL
+# insertion at the AddrLivenessGroupSize boundary (default 16).
---
-name: multi_load_kill
+name: flush_group_kill
tracksRegLiveness: true
+registers:
+ - { id: 0, class: vgpr_32 }
+ - { id: 1, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 2, class: vgpr_32 }
+ - { id: 3, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 4, class: vgpr_32 }
+ - { id: 5, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 6, class: vgpr_32 }
+ - { id: 7, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 8, class: vgpr_32 }
+ - { id: 9, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 10, class: vgpr_32 }
+ - { id: 11, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 12, class: vgpr_32 }
+ - { id: 13, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 14, class: vgpr_32 }
+ - { id: 15, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 16, class: vgpr_32 }
+ - { id: 17, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 18, class: vgpr_32 }
+ - { id: 19, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 20, class: vgpr_32 }
+ - { id: 21, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 22, class: vgpr_32 }
+ - { id: 23, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 24, class: vgpr_32 }
+ - { id: 25, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 26, class: vgpr_32 }
+ - { id: 27, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 28, class: vgpr_32 }
+ - { id: 29, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 30, class: vgpr_32 }
+ - { id: 31, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 32, class: vgpr_32 }
+ - { id: 33, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 34, class: vgpr_32 }
+ - { id: 35, class: sreg_64_xexec_xnull, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 36, class: vgpr_32 }
body: |
bb.0:
- liveins: $sgpr44_sgpr45
+ liveins: $sgpr0_sgpr1
- ; GFX1250-LABEL: name: multi_load_kill
- ; GFX1250: liveins: $sgpr44_sgpr45
+ ; GFX1250-LABEL: name: flush_group_kill
+ ; GFX1250: liveins: $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: dead [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]]
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
- ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX1250-NEXT: $sgpr45 = S_MOV_B32 0
- ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX1250-NEXT: dead [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[COPY2]]
- ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY3]], 0, 0, implicit $exec, implicit $exec :: (load (s32), addrspace 1)
- ; GFX1250-NEXT: $sgpr44 = S_MOV_B32 1
- ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]]
+ ; GFX1250-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM1]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM2:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 16, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM2]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD2:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY2]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM3:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 24, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM3]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD3:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY3]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM4:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 32, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM4]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD4:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY4]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM5:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM5]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD5:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY5]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM6:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 48, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM6]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD6:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY6]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM7:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 56, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM7]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD7:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY7]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM8:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 64, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM8]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD8:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY8]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM9:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 72, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM9]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD9:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY9]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM10:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 80, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM10]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD10:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY10]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM11:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 88, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM11]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD11:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY11]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM12:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 96, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM12]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD12:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY12]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM13:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 104, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM13]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD13:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY13]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM14:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 112, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM14]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD14:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY14]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM15:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 120, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM15]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD15:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY15]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: KILL [[COPY]], [[COPY1]], [[COPY2]], [[COPY3]], [[COPY4]], [[COPY5]], [[COPY6]], [[COPY7]], [[COPY8]], [[COPY9]], [[COPY10]], [[COPY11]], [[COPY12]], [[COPY13]], [[COPY14]], [[COPY15]]
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM16:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[S_LOAD_DWORDX2_IMM16]]
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD16:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY16]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: [[S_LOAD_DWORDX2_IMM17:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 136, 0 :: (dereferenceable invariant load (s64))
+ ; GFX1250-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM17]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX1250-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD]], implicit [[GLOBAL_LOAD_DWORD1]], implicit [[GLOBAL_LOAD_DWORD2]], implicit [[GLOBAL_LOAD_DWORD3]], implicit [[GLOBAL_LOAD_DWORD4]], implicit [[GLOBAL_LOAD_DWORD5]], implicit [[GLOBAL_LOAD_DWORD6]], implicit [[GLOBAL_LOAD_DWORD7]], implicit [[GLOBAL_LOAD_DWORD8]], implicit [[GLOBAL_LOAD_DWORD9]], implicit [[GLOBAL_LOAD_DWORD10]], implicit [[GLOBAL_LOAD_DWORD11]], implicit [[GLOBAL_LOAD_DWORD12]], implicit [[GLOBAL_LOAD_DWORD13]], implicit [[GLOBAL_LOAD_DWORD14]], implicit [[GLOBAL_LOAD_DWORD15]], implicit [[GLOBAL_LOAD_DWORD16]], implicit [[GLOBAL_LOAD_DWORD_SADDR]]
;
- ; GFX942-LABEL: name: multi_load_kill
- ; GFX942: liveins: $sgpr44_sgpr45
+ ; GFX942-LABEL: name: flush_group_kill
+ ; GFX942: liveins: $sgpr0_sgpr1
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
- ; GFX942-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; GFX942-NEXT: $sgpr44 = S_MOV_B32 0
- ; GFX942-NEXT: $sgpr45 = S_MOV_B32 0
- ; GFX942-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY1]], [[DEF]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
- ; GFX942-NEXT: $sgpr44 = S_MOV_B32 1
- ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]]
- %0:vgpr_32 = IMPLICIT_DEF
- %3:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- %1:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 0
- $sgpr45 = S_MOV_B32 0
- %4:sreg_64_xexec_xnull = COPY $sgpr44_sgpr45
- %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %4, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
- $sgpr44 = S_MOV_B32 1
- S_ENDPGM 0, implicit %1, implicit %2
+ ; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM2:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 16, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR2:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM2]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM3:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 24, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR3:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM3]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM4:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 32, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR4:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM4]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM5:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR5:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM5]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM6:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 48, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR6:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM6]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM7:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 56, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR7:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM7]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM8:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 64, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR8:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM8]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM9:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 72, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR9:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM9]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM10:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 80, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR10:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM10]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM11:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 88, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR11:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM11]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM12:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 96, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR12:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM12]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM13:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 104, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR13:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM13]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM14:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 112, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR14:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM14]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM15:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 120, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR15:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM15]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM16:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR16:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM16]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: [[S_LOAD_DWORDX2_IMM17:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 136, 0 :: (dereferenceable invariant load (s64))
+ ; GFX942-NEXT: [[GLOBAL_LOAD_DWORD_SADDR17:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[S_LOAD_DWORDX2_IMM17]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ ; GFX942-NEXT: S_ENDPGM 0, implicit [[GLOBAL_LOAD_DWORD_SADDR]], implicit [[GLOBAL_LOAD_DWORD_SADDR1]], implicit [[GLOBAL_LOAD_DWORD_SADDR2]], implicit [[GLOBAL_LOAD_DWORD_SADDR3]], implicit [[GLOBAL_LOAD_DWORD_SADDR4]], implicit [[GLOBAL_LOAD_DWORD_SADDR5]], implicit [[GLOBAL_LOAD_DWORD_SADDR6]], implicit [[GLOBAL_LOAD_DWORD_SADDR7]], implicit [[GLOBAL_LOAD_DWORD_SADDR8]], implicit [[GLOBAL_LOAD_DWORD_SADDR9]], implicit [[GLOBAL_LOAD_DWORD_SADDR10]], implicit [[GLOBAL_LOAD_DWORD_SADDR11]], implicit [[GLOBAL_LOAD_DWORD_SADDR12]], implicit [[GLOBAL_LOAD_DWORD_SADDR13]], implicit [[GLOBAL_LOAD_DWORD_SADDR14]], implicit [[GLOBAL_LOAD_DWORD_SADDR15]], implicit [[GLOBAL_LOAD_DWORD_SADDR16]], implicit [[GLOBAL_LOAD_DWORD_SADDR17]]
+ %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64))
+ %2:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %1, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %3:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64))
+ %4:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %3, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %5:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 16, 0 :: (dereferenceable invariant load (s64))
+ %6:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %5, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %7:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 24, 0 :: (dereferenceable invariant load (s64))
+ %8:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %7, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %9:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 32, 0 :: (dereferenceable invariant load (s64))
+ %10:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %9, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %11:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 40, 0 :: (dereferenceable invariant load (s64))
+ %12:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %11, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %13:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 48, 0 :: (dereferenceable invariant load (s64))
+ %14:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %13, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %15:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 56, 0 :: (dereferenceable invariant load (s64))
+ %16:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %15, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %17:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 64, 0 :: (dereferenceable invariant load (s64))
+ %18:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %17, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %19:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 72, 0 :: (dereferenceable invariant load (s64))
+ %20:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %19, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %21:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 80, 0 :: (dereferenceable invariant load (s64))
+ %22:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %21, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %23:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 88, 0 :: (dereferenceable invariant load (s64))
+ %24:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %23, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %25:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 96, 0 :: (dereferenceable invariant load (s64))
+ %26:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %25, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %27:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 104, 0 :: (dereferenceable invariant load (s64))
+ %28:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %27, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %29:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 112, 0 :: (dereferenceable invariant load (s64))
+ %30:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %29, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %31:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 120, 0 :: (dereferenceable invariant load (s64))
+ %32:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %31, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %33:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s64))
+ %34:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %33, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ %35:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 136, 0 :: (dereferenceable invariant load (s64))
+ %36:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %35, %0, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+ S_ENDPGM 0, implicit %2, implicit %4, implicit %6, implicit %8, implicit %10, implicit %12, implicit %14, implicit %16, implicit %18, implicit %20, implicit %22, implicit %24, implicit %26, implicit %28, implicit %30, implicit %32, implicit %34, implicit %36
...
>From 7f187561ad9af6a875ce7496fc52f988028e2d52 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Wed, 8 Apr 2026 14:53:58 -0500
Subject: [PATCH 16/16] typo & merge test update
---
llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
index 890e21a050a0f..d532727d7b257 100644
--- a/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixXcntStallSAddrReuse.cpp
@@ -248,10 +248,10 @@ MachineInstr *SIFixXcntStallSAddrReuse::convertToVAddr(MachineInstr &MI,
LIS->InsertMachineInstrInMaps(*AddLo);
LIS->InsertMachineInstrInMaps(*AddHi);
LIS->createAndComputeVirtRegInterval(TmpVAddr);
- LIS->createAndComputeVirtRegInterval(VAddrSignExt);
+ LIS->createAndComputeVirtRegInterval(HiExt);
LIS->createAndComputeVirtRegInterval(SAddrV);
- LIS->createAndComputeVirtRegInterval(SAddrLo);
- LIS->createAndComputeVirtRegInterval(SAddrHi);
+ LIS->createAndComputeVirtRegInterval(LoV);
+ LIS->createAndComputeVirtRegInterval(HiV);
}
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(NewOpc));
More information about the llvm-commits
mailing list