[llvm] [AMDGPU] Rewrite SADDR memory ops to VADDR reduce xcnt stalls (PR #190654)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 23:01:43 PDT 2026
================
@@ -0,0 +1,277 @@
+//===- SIFixXcntStallSAddrReuse.cpp - Fix xcnt stalls from SADDR reuse ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// Convert FLAT global SADDR memory ops to VADDR when the physical SGPR
+/// address pair is redefined later in the block. This avoids s_wait_xcnt
+/// stalls from XNACK replay hazards on reused SGPR pairs.
+///
+/// Runs after SGPR RA and before VGPR RA.
+//===----------------------------------------------------------------------===//
+
+#include "SIFixXcntStallSAddrReuse.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-fix-xcnt-stall-saddr-reuse"
+
+STATISTIC(NumConverted, "Number of SADDR loads converted to VADDR");
+
+namespace {
+
+class SIFixXcntStallSAddrReuse {
+ const SIInstrInfo *TII = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+ MachineRegisterInfo *MRI = nullptr;
+ LiveIntervals *LIS = nullptr;
+
+ bool sAddrRedefinedAfter(MachineInstr &MI, Register SAddr);
+ MachineInstr *convertToVAddr(MachineInstr &MI, MachineBasicBlock &MBB,
+ Register &NewAddrReg);
+ bool processMBB(MachineBasicBlock &MBB);
+
+public:
+ SIFixXcntStallSAddrReuse(LiveIntervals *LIS = nullptr) : LIS(LIS) {}
+ bool run(MachineFunction &MF);
+};
+
+class SIFixXcntStallSAddrReuseLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+ SIFixXcntStallSAddrReuseLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
+ return false;
+ auto *LISWrapper = getAnalysisIfAvailable<llvm::LiveIntervalsWrapperPass>();
+ llvm::LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
+ return SIFixXcntStallSAddrReuse(LIS).run(MF);
+ }
+
+ StringRef getPassName() const override {
+ return "SI Fix Xcnt Stall SAddr Reuse";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+
+} // end anonymous namespace
+
+char SIFixXcntStallSAddrReuseLegacy::ID = 0;
+
+INITIALIZE_PASS(SIFixXcntStallSAddrReuseLegacy, DEBUG_TYPE,
+ "SI Fix Xcnt Stall SAddr Reuse", false, false)
+
+char &llvm::SIFixXcntStallSAddrReuseLegacyID =
+ SIFixXcntStallSAddrReuseLegacy::ID;
+
+PreservedAnalyses
+SIFixXcntStallSAddrReusePass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ if (!MF.getSubtarget<GCNSubtarget>().hasWaitXcnt())
+ return PreservedAnalyses::all();
+ auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
+ if (!SIFixXcntStallSAddrReuse(LIS).run(MF))
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ if (LIS)
+ PA.preserve<LiveIntervalsAnalysis>();
+ return PA;
+}
+
+static bool isEligible(const MachineInstr &MI, const SIInstrInfo &TII,
+ const SIRegisterInfo &TRI,
+ const MachineRegisterInfo &MRI, int &SAddrIdx) {
+ SAddrIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::saddr);
+ if (SAddrIdx < 0 || !TII.isFLATGlobal(MI))
+ return false;
+ const MachineOperand &SAddr = MI.getOperand(SAddrIdx);
+ return SAddr.isReg() && SAddr.getReg().isPhysical() &&
+ TRI.isSGPRReg(MRI, SAddr.getReg());
+}
+
+bool SIFixXcntStallSAddrReuse::sAddrRedefinedAfter(MachineInstr &MI,
+ Register SAddr) {
+ MachineBasicBlock::iterator I = std::next(MI.getIterator());
+ MachineBasicBlock::iterator E = MI.getParent()->end();
+ for (; I != E; ++I) {
----------------
arsenm wrote:
Raneg loop with make_range?
https://github.com/llvm/llvm-project/pull/190654
More information about the llvm-commits
mailing list