[llvm] [AMDGPU] Add SSA-form memory clause pass (SSASIFormMemoryClauses) (PR #209656)
Jun Wang via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 27 18:18:20 PDT 2026
https://github.com/jwanggit86 updated https://github.com/llvm/llvm-project/pull/209656
>From f3b80bb3dd0fbaf440988b83141fe1b75deed1d5 Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86 at yahoo.com>
Date: Wed, 24 Jun 2026 16:19:31 -0700
Subject: [PATCH 1/5] Version 1; no tests yet; This is incorect because it
relies on live interval analysis which is a post-SSA pass.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 4 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 19 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
.../Target/AMDGPU/SSASIFormMemoryClauses.cpp | 441 ++++++++++++++++++
.../Target/AMDGPU/SSASIFormMemoryClauses.h | 23 +
5 files changed, 487 insertions(+), 1 deletion(-)
create mode 100644 llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
create mode 100644 llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c6dd1dbb62449..bf52cd32b1a34 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -55,6 +55,7 @@ FunctionPass *createSIMemoryLegalizerPass();
FunctionPass *createSIInsertWaitcntsPass();
FunctionPass *createSIPreAllocateWWMRegsLegacyPass();
FunctionPass *createSIFormMemoryClausesLegacyPass();
+FunctionPass *createSSASIFormMemoryClausesLegacyPass();
FunctionPass *createSIPostRABundlerPass();
FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
@@ -551,6 +552,9 @@ extern char &SIInsertWaitcntsID;
void initializeSIFormMemoryClausesLegacyPass(PassRegistry &);
extern char &SIFormMemoryClausesID;
+void initializeSSASIFormMemoryClausesLegacyPass(PassRegistry &);
+extern char &SSASIFormMemoryClausesID;
+
void initializeSIPostRABundlerLegacyPass(PassRegistry &);
extern char &SIPostRABundlerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index ae6e6d0bdcd1e..71c3a8c8848ab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -53,6 +53,7 @@
#include "SIFixVGPRCopies.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
+#include "SSASIFormMemoryClauses.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -564,6 +565,11 @@ static cl::opt<bool> EnablePreRAOptimizations(
cl::desc("Enable Pre-RA optimizations pass"), cl::init(true),
cl::Hidden);
+static cl::opt<bool> EnableSSASIFormMemoryClauses(
+ "amdgpu-enable-ssa-form-memory-clauses",
+ cl::desc("Enable SSA form memory clause pass (before PHI elimination)"),
+ cl::init(false), cl::Hidden);
+
static cl::opt<bool> EnablePromoteKernelArguments(
"amdgpu-enable-promote-kernel-arguments",
cl::desc("Enable promotion of flat kernel pointer arguments to global"),
@@ -714,6 +720,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSIOptimizeExecMaskingLegacyPass(*PR);
initializeSIPreAllocateWWMRegsLegacyPass(*PR);
initializeSIFormMemoryClausesLegacyPass(*PR);
+ initializeSSASIFormMemoryClausesLegacyPass(*PR);
initializeSIPostRABundlerLegacyPass(*PR);
initializeGCNCreateVOPDLegacyPass(*PR);
initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
@@ -1413,6 +1420,7 @@ AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
// Garbage collection is not supported.
disablePass(&GCLoweringID);
disablePass(&ShadowStackGCLoweringID);
+
}
void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
@@ -1758,9 +1766,18 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// This is not an essential optimization and it has a noticeable impact on
// compilation time, so we only enable it from O2.
- if (TM->getOptLevel() > CodeGenOptLevel::Less)
+ if (TM->getOptLevel() > CodeGenOptLevel::Less && !EnableSSASIFormMemoryClauses)
insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
+ // Run the SSA form of the memory clause pass before PHI elimination.
+ // LiveIntervals is required and inserted immediately before the pass.
+ // TODO: Once PR #161054 (SSAMachineScheduler) is merged, anchor this pass
+ // after SSAMachineSchedulerID instead of LiveVariablesID.
+ if (EnableSSASIFormMemoryClauses) {
+ insertPass(&LiveVariablesID, &LiveIntervalsID);
+ insertPass(&LiveIntervalsID, &SSASIFormMemoryClausesID);
+ }
+
TargetPassConfig::addOptimizedRegAlloc();
}
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 46edc44e2cc05..8273851731b17 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -163,6 +163,7 @@ add_llvm_target(AMDGPUCodeGen
SIFixVGPRCopies.cpp
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
+ SSASIFormMemoryClauses.cpp
SIFrameLowering.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
new file mode 100644
index 0000000000000..2751c06c87332
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
@@ -0,0 +1,441 @@
+//===-- SSASIFormMemoryClauses.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This pass is a clone of SIFormMemoryClauses intended to run in SSA
+/// form, before PHI elimination. It extends the live ranges of registers used
+/// as pointers in sequences of adjacent SMEM and VMEM instructions when XNACK
+/// is enabled, preventing a load from overwriting a pointer and requiring a
+/// soft clause break.
+///
+/// TODO: Once PR #161054 (SSAMachineScheduler) is merged this pass should be
+/// placed immediately after SSAMachineScheduler in the pipeline.
+///
+//===----------------------------------------------------------------------===//
+
+#include "SSASIFormMemoryClauses.h"
+#include "AMDGPU.h"
+#include "GCNRegPressure.h"
+#include "SIMachineFunctionInfo.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "ssa-si-form-memory-clauses"
+
+// Clauses longer then 15 instructions would overflow one of the counters
+// and stall. They can stall even earlier if there are outstanding counters.
+static cl::opt<unsigned>
+SSAMaxClause("amdgpu-ssa-max-memory-clause", cl::Hidden, cl::init(15),
+ cl::desc("Maximum length of a memory clause for SSA form pass, "
+ "instructions"));
+
+namespace {
+
+class SSASIFormMemoryClausesImpl {
+ using RegUse = DenseMap<unsigned, std::pair<RegState, LaneBitmask>>;
+
+ bool canBundle(const MachineInstr &MI, const RegUse &Defs,
+ const RegUse &Uses) const;
+ bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
+ void collectRegUses(const MachineInstr &MI, RegUse &Defs,
+ RegUse &Uses) const;
+ bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
+ GCNDownwardRPTracker &RPT);
+
+ const GCNSubtarget *ST;
+ const SIRegisterInfo *TRI;
+ const MachineRegisterInfo *MRI;
+ SIMachineFunctionInfo *MFI;
+ LiveIntervals *LIS;
+
+ unsigned LastRecordedOccupancy;
+ unsigned MaxVGPRs;
+ unsigned MaxSGPRs;
+
+public:
+ SSASIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
+ bool run(MachineFunction &MF);
+};
+
+class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SSASIFormMemoryClausesLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override {
+ return "SSA SI Form memory clauses";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<LiveIntervalsWrapperPass>();
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ // Unlike SIFormMemoryClauses, we do NOT clear the IsSSA property because
+ // this pass is designed to run while the function is still in SSA form.
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
+ "SSA SI Form memory clauses", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
+INITIALIZE_PASS_END(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
+ "SSA SI Form memory clauses", false, false)
+
+char SSASIFormMemoryClausesLegacy::ID = 0;
+
+char &llvm::SSASIFormMemoryClausesID = SSASIFormMemoryClausesLegacy::ID;
+
+FunctionPass *llvm::createSSASIFormMemoryClausesLegacyPass() {
+ return new SSASIFormMemoryClausesLegacy();
+}
+
+static bool isVMEMClauseInst(const MachineInstr &MI) {
+ return SIInstrInfo::isVMEM(MI);
+}
+
+static bool isSMEMClauseInst(const MachineInstr &MI) {
+ return SIInstrInfo::isSMRD(MI);
+}
+
+// There no sense to create store clauses, they do not define anything,
+// thus there is nothing to set early-clobber.
+static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
+ assert(!MI.isDebugInstr() && "debug instructions should not reach here");
+ if (MI.isBundled())
+ return false;
+ if (!MI.mayLoad() || MI.mayStore())
+ return false;
+ if (SIInstrInfo::isAtomic(MI))
+ return false;
+ if (IsVMEMClause && !isVMEMClauseInst(MI))
+ return false;
+ if (!IsVMEMClause && !isSMEMClauseInst(MI))
+ return false;
+ // If this is a load instruction where the result has been coalesced with an
+ // operand, then we cannot clause it.
+ for (const MachineOperand &ResMO : MI.defs()) {
+ Register ResReg = ResMO.getReg();
+ for (const MachineOperand &MO : MI.all_uses()) {
+ if (MO.getReg() == ResReg)
+ return false;
+ }
+ break; // Only check the first def.
+ }
+ return true;
+}
+
+static RegState getMopState(const MachineOperand &MO) {
+ RegState S = {};
+ if (MO.isImplicit())
+ S |= RegState::Implicit;
+ if (MO.isDead())
+ S |= RegState::Dead;
+ if (MO.isUndef())
+ S |= RegState::Undef;
+ if (MO.isKill())
+ S |= RegState::Kill;
+ if (MO.isEarlyClobber())
+ S |= RegState::EarlyClobber;
+ if (MO.getReg().isPhysical() && MO.isRenamable())
+ S |= RegState::Renamable;
+ return S;
+}
+
+// Returns false if there is a use of a def already in the map.
+// In this case we must break the clause.
+bool SSASIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
+ const RegUse &Defs,
+ const RegUse &Uses) const {
+ // Check interference with defs.
+ for (const MachineOperand &MO : MI.operands()) {
+ // TODO: Prologue/Epilogue Insertion pass does not process bundled
+ // instructions.
+ if (MO.isFI())
+ return false;
+
+ if (!MO.isReg())
+ continue;
+
+ Register Reg = MO.getReg();
+
+ // If it is tied we will need to write same register as we read.
+ if (MO.isTied())
+ return false;
+
+ const RegUse &Map = MO.isDef() ? Uses : Defs;
+ auto Conflict = Map.find(Reg);
+ if (Conflict == Map.end())
+ continue;
+
+ if (Reg.isPhysical())
+ return false;
+
+ LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
+ if ((Conflict->second.second & Mask).any())
+ return false;
+ }
+
+ return true;
+}
+
+// Since all defs in the clause are early clobber we can run out of registers.
+// Function returns false if pressure would hit the limit if instruction is
+// bundled into a memory clause.
+bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
+ GCNDownwardRPTracker &RPT) {
+ // NB: skip advanceBeforeNext() call. Since all defs will be marked
+ // early-clobber they will all stay alive at least to the end of the
+ // clause. Therefor we should not decrease pressure even if load
+ // pointer becomes dead and could otherwise be reused for destination.
+ RPT.advanceToNext();
+ GCNRegPressure MaxPressure = RPT.moveMaxPressure();
+ unsigned Occupancy = MaxPressure.getOccupancy(
+ *ST,
+ MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
+
+ // Don't push over half the register budget. We don't want to introduce
+ // spilling just to form a soft clause.
+ //
+ // FIXME: This pressure check is fundamentally broken. First, this is checking
+ // the global pressure, not the pressure at this specific point in the
+ // program. Second, it's not accounting for the increased liveness of the use
+ // operands due to the early clobber we will introduce. Third, the pressure
+ // tracking does not account for the alignment requirements for SGPRs, or the
+ // fragmentation of registers the allocator will need to satisfy.
+ if (Occupancy >= MFI->getMinAllowedOccupancy() &&
+ MaxPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
+ MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
+ LastRecordedOccupancy = Occupancy;
+ return true;
+ }
+ return false;
+}
+
+// Collect register defs and uses along with their lane masks and states.
+void SSASIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
+ RegUse &Defs,
+ RegUse &Uses) const {
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg())
+ continue;
+ Register Reg = MO.getReg();
+ if (!Reg)
+ continue;
+
+ LaneBitmask Mask = Reg.isVirtual()
+ ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
+ : LaneBitmask::getAll();
+ RegUse &Map = MO.isDef() ? Defs : Uses;
+
+ RegState State = getMopState(MO);
+ auto [Loc, Inserted] = Map.try_emplace(Reg, State, Mask);
+ if (!Inserted) {
+ Loc->second.first |= State;
+ Loc->second.second |= Mask;
+ }
+ }
+}
+
+// Check register def/use conflicts, occupancy limits and collect def/use maps.
+// Return true if instruction can be bundled with previous. If it cannot
+// def/use maps are not updated.
+bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
+ RegUse &Defs, RegUse &Uses,
+ GCNDownwardRPTracker &RPT) {
+ if (!canBundle(MI, Defs, Uses))
+ return false;
+
+ if (!checkPressure(MI, RPT))
+ return false;
+
+ collectRegUses(MI, Defs, Uses);
+ return true;
+}
+
+bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
+ ST = &MF.getSubtarget<GCNSubtarget>();
+ if (!ST->isXNACKEnabled())
+ return false;
+
+ const SIInstrInfo *TII = ST->getInstrInfo();
+ TRI = ST->getRegisterInfo();
+ MRI = &MF.getRegInfo();
+ MFI = MF.getInfo<SIMachineFunctionInfo>();
+ SlotIndexes *Ind = LIS->getSlotIndexes();
+ bool Changed = false;
+
+ MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
+ MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
+ unsigned FuncMaxClause = MF.getFunction().getFnAttributeAsParsedInteger(
+ "amdgpu-max-memory-clause", SSAMaxClause);
+
+ for (MachineBasicBlock &MBB : MF) {
+ GCNDownwardRPTracker RPT(*LIS);
+ MachineBasicBlock::instr_iterator Next;
+ for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
+ MachineInstr &MI = *I;
+ Next = std::next(I);
+
+ if (MI.isMetaInstruction())
+ continue;
+
+ bool IsVMEM = isVMEMClauseInst(MI);
+
+ if (!isValidClauseInst(MI, IsVMEM))
+ continue;
+
+ if (!RPT.getNext().isValid())
+ RPT.reset(MI, MBB.end());
+ else { // Advance the state to the current MI.
+ RPT.advance(MachineBasicBlock::const_iterator(MI));
+ RPT.advanceBeforeNext();
+ }
+
+ const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
+ RegUse Defs, Uses;
+ if (!processRegUses(MI, Defs, Uses, RPT)) {
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ continue;
+ }
+
+ MachineBasicBlock::iterator LastClauseInst = Next;
+ unsigned Length = 1;
+ for (; Next != E && Length < FuncMaxClause; ++Next) {
+ // Debug instructions should not change the kill insertion.
+ if (Next->isMetaInstruction())
+ continue;
+
+ if (!isValidClauseInst(*Next, IsVMEM))
+ break;
+
+ // A load from pointer which was loaded inside the same bundle is an
+ // impossible clause because we will need to write and read the same
+ // register inside. In this case processRegUses will return false.
+ if (!processRegUses(*Next, Defs, Uses, RPT))
+ break;
+
+ LastClauseInst = Next;
+ ++Length;
+ }
+ if (Length < 2) {
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ continue;
+ }
+
+ Changed = true;
+ MFI->limitOccupancy(LastRecordedOccupancy);
+
+ assert(!LastClauseInst->isMetaInstruction());
+
+ SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(MI);
+ SlotIndex ClauseLiveOutIdx =
+ LIS->getInstructionIndex(*LastClauseInst).getNextIndex();
+
+ // Track the last inserted kill.
+ MachineInstrBuilder Kill;
+
+ // Insert one kill per register, with operands covering all necessary
+ // subregisters.
+ for (auto &&R : Uses) {
+ Register Reg = R.first;
+ if (Reg.isPhysical())
+ continue;
+
+ // Collect the register operands we should extend the live ranges of.
+ SmallVector<std::tuple<RegState, unsigned>> KillOps;
+ const LiveInterval &LI = LIS->getInterval(R.first);
+
+ if (!LI.hasSubRanges()) {
+ if (!LI.liveAt(ClauseLiveOutIdx)) {
+ KillOps.emplace_back(R.second.first | RegState::Kill,
+ AMDGPU::NoSubRegister);
+ }
+ } else {
+ LaneBitmask KilledMask;
+ for (const LiveInterval::SubRange &SR : LI.subranges()) {
+ if (SR.liveAt(ClauseLiveInIdx) && !SR.liveAt(ClauseLiveOutIdx))
+ KilledMask |= SR.LaneMask;
+ }
+
+ if (KilledMask.none())
+ continue;
+
+ SmallVector<unsigned> KilledIndexes;
+ bool Success = TRI->getCoveringSubRegIndexes(
+ MRI->getRegClass(Reg), KilledMask, KilledIndexes);
+ (void)Success;
+ assert(Success && "Failed to find subregister mask to cover lanes");
+ for (unsigned SubReg : KilledIndexes) {
+ KillOps.emplace_back(R.second.first | RegState::Kill, SubReg);
+ }
+ }
+
+ if (KillOps.empty())
+ continue;
+
+ // We only want to extend the live ranges of used registers. If they
+ // already have existing uses beyond the bundle, we don't need the kill.
+ //
+ // It's possible all of the use registers were already live past the
+ // bundle.
+ Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst),
+ DebugLoc(), TII->get(AMDGPU::KILL));
+ for (auto &Op : KillOps)
+ Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
+ Ind->insertMachineInstrInMaps(*Kill);
+ }
+
+ // Restore the state after processing the end of the bundle.
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+
+ if (!Kill)
+ continue;
+
+ for (auto &&R : Defs) {
+ Register Reg = R.first;
+ Uses.erase(Reg);
+ if (Reg.isPhysical())
+ continue;
+ LIS->removeInterval(Reg);
+ LIS->createAndComputeVirtRegInterval(Reg);
+ }
+
+ for (auto &&R : Uses) {
+ Register Reg = R.first;
+ if (Reg.isPhysical())
+ continue;
+ LIS->removeInterval(Reg);
+ LIS->createAndComputeVirtRegInterval(Reg);
+ }
+ }
+ }
+
+ return Changed;
+}
+
+bool SSASIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+
+ LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+ return SSASIFormMemoryClausesImpl(LIS).run(MF);
+}
+
+PreservedAnalyses
+SSASIFormMemoryClausesPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SSASIFormMemoryClausesImpl(&LIS).run(MF);
+ return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h
new file mode 100644
index 0000000000000..099b975623225
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h
@@ -0,0 +1,23 @@
+//===- SSASIFormMemoryClauses.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
+#define LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class SSASIFormMemoryClausesPass
+ : public OptionalPassInfoMixin<SSASIFormMemoryClausesPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
>From fc44999ac467779203807d96b09d22bd8d8e9edf Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86 at yahoo.com>
Date: Tue, 7 Jul 2026 20:06:00 -0700
Subject: [PATCH 2/5] Remove reliance on LiveInterval; track reg pressure at
the block level, which may not be correct
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 +-
.../Target/AMDGPU/SSASIFormMemoryClauses.cpp | 215 ++++++++++++------
2 files changed, 144 insertions(+), 80 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 71c3a8c8848ab..a34a69898ad31 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1770,13 +1770,12 @@ void GCNPassConfig::addOptimizedRegAlloc() {
insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
// Run the SSA form of the memory clause pass before PHI elimination.
- // LiveIntervals is required and inserted immediately before the pass.
+ // LiveVariables is the anchor: it runs in SSA form and sets kill flags that
+ // our pass relies on for intra-block liveness tracking.
// TODO: Once PR #161054 (SSAMachineScheduler) is merged, anchor this pass
// after SSAMachineSchedulerID instead of LiveVariablesID.
- if (EnableSSASIFormMemoryClauses) {
- insertPass(&LiveVariablesID, &LiveIntervalsID);
- insertPass(&LiveIntervalsID, &SSASIFormMemoryClausesID);
- }
+ if (EnableSSASIFormMemoryClauses)
+ insertPass(&LiveVariablesID, &SSASIFormMemoryClausesID);
TargetPassConfig::addOptimizedRegAlloc();
}
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
index 2751c06c87332..35ea33e5e3cff 100644
--- a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
@@ -41,24 +41,28 @@ class SSASIFormMemoryClausesImpl {
bool canBundle(const MachineInstr &MI, const RegUse &Defs,
const RegUse &Uses) const;
- bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
+ bool checkPressure(const MachineInstr &MI, GCNRegPressure &CurPressure);
void collectRegUses(const MachineInstr &MI, RegUse &Defs,
RegUse &Uses) const;
bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
- GCNDownwardRPTracker &RPT);
+ GCNRegPressure &CurPressure);
+
+ /// Returns true if \p Reg (masked by \p Mask) has any use strictly after
+ /// \p AfterIt in \p MBB, ignoring meta instructions.
+ bool hasUseAfter(Register Reg, LaneBitmask Mask,
+ MachineBasicBlock::instr_iterator AfterIt,
+ const MachineBasicBlock &MBB) const;
const GCNSubtarget *ST;
const SIRegisterInfo *TRI;
const MachineRegisterInfo *MRI;
SIMachineFunctionInfo *MFI;
- LiveIntervals *LIS;
unsigned LastRecordedOccupancy;
unsigned MaxVGPRs;
unsigned MaxSGPRs;
public:
- SSASIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
bool run(MachineFunction &MF);
};
@@ -75,7 +79,6 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<LiveIntervalsWrapperPass>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -88,7 +91,6 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
INITIALIZE_PASS_BEGIN(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
-INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
INITIALIZE_PASS_END(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
@@ -192,15 +194,28 @@ bool SSASIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
// Since all defs in the clause are early clobber we can run out of registers.
// Function returns false if pressure would hit the limit if instruction is
// bundled into a memory clause.
+//
+// We accumulate pressure monotonically across the clause: because all defs are
+// marked early-clobber they remain live until the clause end, so we never
+// subtract pressure for uses that die mid-clause. This is conservative and
+// avoids the need for LiveIntervals.
bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
- GCNDownwardRPTracker &RPT) {
- // NB: skip advanceBeforeNext() call. Since all defs will be marked
- // early-clobber they will all stay alive at least to the end of the
- // clause. Therefor we should not decrease pressure even if load
- // pointer becomes dead and could otherwise be reused for destination.
- RPT.advanceToNext();
- GCNRegPressure MaxPressure = RPT.moveMaxPressure();
- unsigned Occupancy = MaxPressure.getOccupancy(
+ GCNRegPressure &CurPressure) {
+ // Speculatively add this instruction's virtual defs to the running pressure.
+ // Physical register defs are skipped: they are not allocatable slots and
+ // GCNRegPressure::inc() requires a virtual register.
+ GCNRegPressure NewPressure = CurPressure;
+ for (const MachineOperand &MO : MI.defs()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ LaneBitmask Mask = MO.getSubReg()
+ ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
+ : MRI->getMaxLaneMaskForVReg(Reg);
+ NewPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
+ }
+
+ unsigned Occupancy = NewPressure.getOccupancy(
*ST,
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
@@ -214,9 +229,10 @@ bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
// tracking does not account for the alignment requirements for SGPRs, or the
// fragmentation of registers the allocator will need to satisfy.
if (Occupancy >= MFI->getMinAllowedOccupancy() &&
- MaxPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
- MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
+ NewPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
+ NewPressure.getSGPRNum() <= MaxSGPRs / 2) {
LastRecordedOccupancy = Occupancy;
+ CurPressure = NewPressure;
return true;
}
return false;
@@ -252,17 +268,39 @@ void SSASIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
// def/use maps are not updated.
bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
RegUse &Defs, RegUse &Uses,
- GCNDownwardRPTracker &RPT) {
+ GCNRegPressure &CurPressure) {
if (!canBundle(MI, Defs, Uses))
return false;
- if (!checkPressure(MI, RPT))
+ if (!checkPressure(MI, CurPressure))
return false;
collectRegUses(MI, Defs, Uses);
return true;
}
+bool SSASIFormMemoryClausesImpl::hasUseAfter(
+ Register Reg, LaneBitmask Mask,
+ MachineBasicBlock::instr_iterator AfterIt,
+ const MachineBasicBlock &MBB) const {
+ for (MachineBasicBlock::const_instr_iterator I = std::next(AfterIt),
+ E = MBB.instr_end();
+ I != E; ++I) {
+ if (I->isMetaInstruction())
+ continue;
+ for (const MachineOperand &MO : I->operands()) {
+ if (!MO.isReg() || MO.getReg() != Reg)
+ continue;
+ LaneBitmask MOLanes = Reg.isVirtual()
+ ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
+ : LaneBitmask::getAll();
+ if ((MOLanes & Mask).any())
+ return true;
+ }
+ }
+ return false;
+}
+
bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
if (!ST->isXNACKEnabled())
@@ -272,7 +310,6 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
TRI = ST->getRegisterInfo();
MRI = &MF.getRegInfo();
MFI = MF.getInfo<SIMachineFunctionInfo>();
- SlotIndexes *Ind = LIS->getSlotIndexes();
bool Changed = false;
MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
@@ -281,7 +318,41 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
"amdgpu-max-memory-clause", SSAMaxClause);
for (MachineBasicBlock &MBB : MF) {
- GCNDownwardRPTracker RPT(*LIS);
+ // BlockPressure tracks the approximate register pressure at the current
+ // scan position within MBB. It is updated instruction-by-instruction:
+ // virtual register defs increase pressure; uses with kill flags decrease
+ // it. In SSA form, kill flags are reliable (each vreg has exactly one
+ // def), so this gives accurate intra-block liveness without LiveIntervals.
+ // Virtual register live-ins from predecessor blocks are not counted (they
+ // would require liveness analysis to enumerate), so BlockPressure
+ // underestimates pressure at block entry but becomes more accurate as more
+ // instructions are processed.
+ GCNRegPressure BlockPressure;
+
+ // PressurePos is the next instruction to be consumed into BlockPressure.
+ // It may lag behind the outer loop iterator when the inner clause-extension
+ // loop advances Next past instructions not admitted to a clause.
+ // advanceBlockPressure() catches it up before each clause attempt.
+ auto PressurePos = MBB.instr_begin();
+
+ auto advanceBlockPressure = [&](MachineBasicBlock::instr_iterator Target) {
+ while (PressurePos != Target) {
+ const MachineInstr &CurMI = *PressurePos++;
+ if (CurMI.isMetaInstruction())
+ continue;
+ for (const MachineOperand &MO : CurMI.operands()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
+ if (MO.isDef())
+ BlockPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
+ else if (MO.isKill())
+ BlockPressure.inc(Reg, Mask, LaneBitmask::getNone(), *MRI);
+ }
+ }
+ };
+
MachineBasicBlock::instr_iterator Next;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
MachineInstr &MI = *I;
@@ -292,24 +363,23 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
bool IsVMEM = isVMEMClauseInst(MI);
- if (!isValidClauseInst(MI, IsVMEM))
+ if (!isValidClauseInst(MI, IsVMEM)) {
+ advanceBlockPressure(Next);
continue;
-
- if (!RPT.getNext().isValid())
- RPT.reset(MI, MBB.end());
- else { // Advance the state to the current MI.
- RPT.advance(MachineBasicBlock::const_iterator(MI));
- RPT.advanceBeforeNext();
}
- const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
+ // Bring BlockPressure up to (but not including) MI, then snapshot it as
+ // the baseline pressure entering this potential clause.
+ advanceBlockPressure(I);
+ GCNRegPressure CurPressure = BlockPressure;
+
RegUse Defs, Uses;
- if (!processRegUses(MI, Defs, Uses, RPT)) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ if (!processRegUses(MI, Defs, Uses, CurPressure)) {
+ advanceBlockPressure(Next);
continue;
}
- MachineBasicBlock::iterator LastClauseInst = Next;
+ MachineBasicBlock::instr_iterator LastClauseInst = Next;
unsigned Length = 1;
for (; Next != E && Length < FuncMaxClause; ++Next) {
// Debug instructions should not change the kill insertion.
@@ -322,14 +392,17 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
// A load from pointer which was loaded inside the same bundle is an
// impossible clause because we will need to write and read the same
// register inside. In this case processRegUses will return false.
- if (!processRegUses(*Next, Defs, Uses, RPT))
+ if (!processRegUses(*Next, Defs, Uses, CurPressure))
break;
LastClauseInst = Next;
++Length;
}
if (Length < 2) {
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+ // Clause did not form; process MI normally. Instructions examined by
+ // the inner loop but not admitted will be caught up by
+ // advanceBlockPressure() at the start of the next outer iteration.
+ advanceBlockPressure(std::next(I));
continue;
}
@@ -338,47 +411,50 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
assert(!LastClauseInst->isMetaInstruction());
- SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(MI);
- SlotIndex ClauseLiveOutIdx =
- LIS->getInstructionIndex(*LastClauseInst).getNextIndex();
-
// Track the last inserted kill.
MachineInstrBuilder Kill;
// Insert one kill per register, with operands covering all necessary
- // subregisters.
+ // subregisters, for use-operands that do not survive past the clause.
+ // Liveness is determined by a local forward scan (reliable in SSA form
+ // because kill flags are accurate) instead of LiveIntervals.
for (auto &&R : Uses) {
Register Reg = R.first;
if (Reg.isPhysical())
continue;
- // Collect the register operands we should extend the live ranges of.
SmallVector<std::tuple<RegState, unsigned>> KillOps;
- const LiveInterval &LI = LIS->getInterval(R.first);
- if (!LI.hasSubRanges()) {
- if (!LI.liveAt(ClauseLiveOutIdx)) {
+ LaneBitmask FullMask = MRI->getMaxLaneMaskForVReg(Reg);
+ if (!MRI->shouldTrackSubRegLiveness(Reg) ||
+ (R.second.second & FullMask) == FullMask) {
+ // Whole-register: insert a kill if no use survives the clause.
+ if (!hasUseAfter(Reg, LaneBitmask::getAll(), LastClauseInst, MBB))
KillOps.emplace_back(R.second.first | RegState::Kill,
AMDGPU::NoSubRegister);
- }
} else {
+ // Per-lane: find which lanes die within the clause.
LaneBitmask KilledMask;
- for (const LiveInterval::SubRange &SR : LI.subranges()) {
- if (SR.liveAt(ClauseLiveInIdx) && !SR.liveAt(ClauseLiveOutIdx))
- KilledMask |= SR.LaneMask;
+ SmallVector<unsigned> SubRegs;
+ bool Success = TRI->getCoveringSubRegIndexes(
+ MRI->getRegClass(Reg), R.second.second, SubRegs);
+ (void)Success;
+ assert(Success && "Failed to find subregister mask to cover lanes");
+ for (unsigned SubReg : SubRegs) {
+ LaneBitmask SubMask = TRI->getSubRegIndexLaneMask(SubReg);
+ if (!hasUseAfter(Reg, SubMask, LastClauseInst, MBB))
+ KilledMask |= SubMask;
}
if (KilledMask.none())
continue;
SmallVector<unsigned> KilledIndexes;
- bool Success = TRI->getCoveringSubRegIndexes(
- MRI->getRegClass(Reg), KilledMask, KilledIndexes);
- (void)Success;
+ Success = TRI->getCoveringSubRegIndexes(MRI->getRegClass(Reg),
+ KilledMask, KilledIndexes);
assert(Success && "Failed to find subregister mask to cover lanes");
- for (unsigned SubReg : KilledIndexes) {
+ for (unsigned SubReg : KilledIndexes)
KillOps.emplace_back(R.second.first | RegState::Kill, SubReg);
- }
}
if (KillOps.empty())
@@ -389,35 +465,26 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
//
// It's possible all of the use registers were already live past the
// bundle.
- Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst),
- DebugLoc(), TII->get(AMDGPU::KILL));
+ Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst), DebugLoc(),
+ TII->get(AMDGPU::KILL));
for (auto &Op : KillOps)
Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
- Ind->insertMachineInstrInMaps(*Kill);
- }
-
- // Restore the state after processing the end of the bundle.
- RPT.reset(MI, MBB.end(), &LiveRegsCopy);
-
- if (!Kill)
- continue;
-
- for (auto &&R : Defs) {
- Register Reg = R.first;
- Uses.erase(Reg);
- if (Reg.isPhysical())
- continue;
- LIS->removeInterval(Reg);
- LIS->createAndComputeVirtRegInterval(Reg);
+ // No SlotIndexes maintenance: SlotIndexes is not live at this point
+ // in the pipeline (pre-PHI elimination).
}
+ // Update BlockPressure to reflect the committed clause. CurPressure
+ // already has all clause defs accumulated. Subtract the uses that die
+ // within the clause (those for which kills were inserted above).
+ BlockPressure = CurPressure;
for (auto &&R : Uses) {
Register Reg = R.first;
if (Reg.isPhysical())
continue;
- LIS->removeInterval(Reg);
- LIS->createAndComputeVirtRegInterval(Reg);
+ if (!hasUseAfter(Reg, R.second.second, LastClauseInst, MBB))
+ BlockPressure.inc(Reg, R.second.second, LaneBitmask::getNone(), *MRI);
}
+ PressurePos = Next;
}
}
@@ -428,14 +495,12 @@ bool SSASIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
- LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
- return SSASIFormMemoryClausesImpl(LIS).run(MF);
+ return SSASIFormMemoryClausesImpl().run(MF);
}
PreservedAnalyses
SSASIFormMemoryClausesPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
- SSASIFormMemoryClausesImpl(&LIS).run(MF);
+ SSASIFormMemoryClausesImpl().run(MF);
return PreservedAnalyses::all();
}
>From 668aa8dad40c9895d50dbd43ff78090ff6ac1b4c Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86 at yahoo.com>
Date: Tue, 14 Jul 2026 16:33:01 -0700
Subject: [PATCH 3/5] Use analysis results of Live Vars; also add tests.
Previously block pressures are initialized to zero. This is not
ideal. Making use of the analysis done by Live Vars (an SSA pass)
we can make the initial block pressure more accurate.
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 6 +-
.../Target/AMDGPU/SSASIFormMemoryClauses.cpp | 191 +++++-------
.../test/CodeGen/AMDGPU/ssa-memory-clause.mir | 281 ++++++++++++++++++
3 files changed, 360 insertions(+), 118 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index a34a69898ad31..d3bce6a2f1764 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -53,7 +53,6 @@
#include "SIFixVGPRCopies.h"
#include "SIFoldOperands.h"
#include "SIFormMemoryClauses.h"
-#include "SSASIFormMemoryClauses.h"
#include "SILoadStoreOptimizer.h"
#include "SILowerControlFlow.h"
#include "SILowerSGPRSpills.h"
@@ -68,6 +67,7 @@
#include "SIPreAllocateWWMRegs.h"
#include "SIShrinkInstructions.h"
#include "SIWholeQuadMode.h"
+#include "SSASIFormMemoryClauses.h"
#include "TargetInfo/AMDGPUTargetInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/Analysis/CGSCCPassManager.h"
@@ -1420,7 +1420,6 @@ AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
// Garbage collection is not supported.
disablePass(&GCLoweringID);
disablePass(&ShadowStackGCLoweringID);
-
}
void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
@@ -1766,7 +1765,8 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// This is not an essential optimization and it has a noticeable impact on
// compilation time, so we only enable it from O2.
- if (TM->getOptLevel() > CodeGenOptLevel::Less && !EnableSSASIFormMemoryClauses)
+ if (TM->getOptLevel() > CodeGenOptLevel::Less &&
+ !EnableSSASIFormMemoryClauses)
insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
// Run the SSA form of the memory clause pass before PHI elimination.
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
index 35ea33e5e3cff..c52422a180f63 100644
--- a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
@@ -21,6 +21,8 @@
#include "AMDGPU.h"
#include "GCNRegPressure.h"
#include "SIMachineFunctionInfo.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/CodeGen/LiveVariables.h"
#include "llvm/InitializePasses.h"
using namespace llvm;
@@ -29,10 +31,10 @@ using namespace llvm;
// Clauses longer then 15 instructions would overflow one of the counters
// and stall. They can stall even earlier if there are outstanding counters.
-static cl::opt<unsigned>
-SSAMaxClause("amdgpu-ssa-max-memory-clause", cl::Hidden, cl::init(15),
- cl::desc("Maximum length of a memory clause for SSA form pass, "
- "instructions"));
+static cl::opt<unsigned> SSAMaxClause(
+ "amdgpu-ssa-max-memory-clause", cl::Hidden, cl::init(15),
+ cl::desc("Maximum length of a memory clause for SSA form pass, "
+ "instructions"));
namespace {
@@ -42,28 +44,22 @@ class SSASIFormMemoryClausesImpl {
bool canBundle(const MachineInstr &MI, const RegUse &Defs,
const RegUse &Uses) const;
bool checkPressure(const MachineInstr &MI, GCNRegPressure &CurPressure);
- void collectRegUses(const MachineInstr &MI, RegUse &Defs,
- RegUse &Uses) const;
+ void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
GCNRegPressure &CurPressure);
- /// Returns true if \p Reg (masked by \p Mask) has any use strictly after
- /// \p AfterIt in \p MBB, ignoring meta instructions.
- bool hasUseAfter(Register Reg, LaneBitmask Mask,
- MachineBasicBlock::instr_iterator AfterIt,
- const MachineBasicBlock &MBB) const;
-
const GCNSubtarget *ST;
const SIRegisterInfo *TRI;
const MachineRegisterInfo *MRI;
SIMachineFunctionInfo *MFI;
+ LiveVariables *LV;
unsigned LastRecordedOccupancy;
unsigned MaxVGPRs;
unsigned MaxSGPRs;
public:
- bool run(MachineFunction &MF);
+ bool run(MachineFunction &MF, LiveVariables &LV);
};
class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
@@ -79,6 +75,7 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<LiveVariablesWrapperPass>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -91,6 +88,7 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
INITIALIZE_PASS_BEGIN(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveVariablesWrapperPass)
INITIALIZE_PASS_END(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
@@ -279,29 +277,7 @@ bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
return true;
}
-bool SSASIFormMemoryClausesImpl::hasUseAfter(
- Register Reg, LaneBitmask Mask,
- MachineBasicBlock::instr_iterator AfterIt,
- const MachineBasicBlock &MBB) const {
- for (MachineBasicBlock::const_instr_iterator I = std::next(AfterIt),
- E = MBB.instr_end();
- I != E; ++I) {
- if (I->isMetaInstruction())
- continue;
- for (const MachineOperand &MO : I->operands()) {
- if (!MO.isReg() || MO.getReg() != Reg)
- continue;
- LaneBitmask MOLanes = Reg.isVirtual()
- ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
- : LaneBitmask::getAll();
- if ((MOLanes & Mask).any())
- return true;
- }
- }
- return false;
-}
-
-bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
+bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
ST = &MF.getSubtarget<GCNSubtarget>();
if (!ST->isXNACKEnabled())
return false;
@@ -310,6 +286,7 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
TRI = ST->getRegisterInfo();
MRI = &MF.getRegInfo();
MFI = MF.getInfo<SIMachineFunctionInfo>();
+ LV = &LVIn;
bool Changed = false;
MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
@@ -318,16 +295,20 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
"amdgpu-max-memory-clause", SSAMaxClause);
for (MachineBasicBlock &MBB : MF) {
- // BlockPressure tracks the approximate register pressure at the current
- // scan position within MBB. It is updated instruction-by-instruction:
+ // BlockPressure tracks the register pressure at the current scan position
+ // within MBB. It is seeded with virtual registers live-in to this block
+ // (as computed by LiveVariables), then updated instruction-by-instruction:
// virtual register defs increase pressure; uses with kill flags decrease
// it. In SSA form, kill flags are reliable (each vreg has exactly one
- // def), so this gives accurate intra-block liveness without LiveIntervals.
- // Virtual register live-ins from predecessor blocks are not counted (they
- // would require liveness analysis to enumerate), so BlockPressure
- // underestimates pressure at block entry but becomes more accurate as more
- // instructions are processed.
+ // def), so this gives accurate intra-block liveness.
GCNRegPressure BlockPressure;
+ for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
+ Register Reg = Register::index2VirtReg(I);
+ if (LV->isLiveIn(Reg, MBB)) {
+ LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(Reg);
+ BlockPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
+ }
+ }
// PressurePos is the next instruction to be consumed into BlockPressure.
// It may lag behind the outer loop iterator when the inner clause-extension
@@ -374,10 +355,27 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
GCNRegPressure CurPressure = BlockPressure;
RegUse Defs, Uses;
+ // Kills: virtual registers with isKill() on any use inside the clause.
+ // These registers die within the clause and need a whole-register KILL
+ // pseudo after the last load to extend their live range past the
+ // early-clobber defs. The specific subreg that LV flagged does not
+ // matter; we always emit a whole-register KILL.
+ DenseSet<Register> Kills;
+
+ auto collectKills = [&](const MachineInstr &Instr) {
+ for (const MachineOperand &MO : Instr.operands()) {
+ if (!MO.isReg() || MO.isDef() || !MO.isKill() ||
+ !MO.getReg().isVirtual())
+ continue;
+ Kills.insert(MO.getReg());
+ }
+ };
+
if (!processRegUses(MI, Defs, Uses, CurPressure)) {
advanceBlockPressure(Next);
continue;
}
+ collectKills(MI);
MachineBasicBlock::instr_iterator LastClauseInst = Next;
unsigned Length = 1;
@@ -395,6 +393,7 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
if (!processRegUses(*Next, Defs, Uses, CurPressure))
break;
+ collectKills(*Next);
LastClauseInst = Next;
++Length;
}
@@ -411,79 +410,39 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
assert(!LastClauseInst->isMetaInstruction());
- // Track the last inserted kill.
- MachineInstrBuilder Kill;
-
- // Insert one kill per register, with operands covering all necessary
- // subregisters, for use-operands that do not survive past the clause.
- // Liveness is determined by a local forward scan (reliable in SSA form
- // because kill flags are accurate) instead of LiveIntervals.
- for (auto &&R : Uses) {
- Register Reg = R.first;
- if (Reg.isPhysical())
- continue;
-
- SmallVector<std::tuple<RegState, unsigned>> KillOps;
-
- LaneBitmask FullMask = MRI->getMaxLaneMaskForVReg(Reg);
- if (!MRI->shouldTrackSubRegLiveness(Reg) ||
- (R.second.second & FullMask) == FullMask) {
- // Whole-register: insert a kill if no use survives the clause.
- if (!hasUseAfter(Reg, LaneBitmask::getAll(), LastClauseInst, MBB))
- KillOps.emplace_back(R.second.first | RegState::Kill,
- AMDGPU::NoSubRegister);
- } else {
- // Per-lane: find which lanes die within the clause.
- LaneBitmask KilledMask;
- SmallVector<unsigned> SubRegs;
- bool Success = TRI->getCoveringSubRegIndexes(
- MRI->getRegClass(Reg), R.second.second, SubRegs);
- (void)Success;
- assert(Success && "Failed to find subregister mask to cover lanes");
- for (unsigned SubReg : SubRegs) {
- LaneBitmask SubMask = TRI->getSubRegIndexLaneMask(SubReg);
- if (!hasUseAfter(Reg, SubMask, LastClauseInst, MBB))
- KilledMask |= SubMask;
- }
-
- if (KilledMask.none())
- continue;
-
- SmallVector<unsigned> KilledIndexes;
- Success = TRI->getCoveringSubRegIndexes(MRI->getRegClass(Reg),
- KilledMask, KilledIndexes);
- assert(Success && "Failed to find subregister mask to cover lanes");
- for (unsigned SubReg : KilledIndexes)
- KillOps.emplace_back(R.second.first | RegState::Kill, SubReg);
- }
-
- if (KillOps.empty())
- continue;
-
- // We only want to extend the live ranges of used registers. If they
- // already have existing uses beyond the bundle, we don't need the kill.
- //
- // It's possible all of the use registers were already live past the
- // bundle.
- Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst), DebugLoc(),
- TII->get(AMDGPU::KILL));
- for (auto &Op : KillOps)
- Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
- // No SlotIndexes maintenance: SlotIndexes is not live at this point
- // in the pipeline (pre-PHI elimination).
+ // For each register killed within the clause, insert a whole-register
+ // KILL pseudo after the clause to extend its liveness through the
+ // early-clobber defs. Registers not in Kills are live past the clause
+ // and need nothing.
+ for (Register Reg : Kills) {
+ auto UseIt = Uses.find(Reg);
+ assert(UseIt != Uses.end());
+ RegState UseState = UseIt->second.first & ~RegState::Kill;
+
+ MachineInstrBuilder Kill =
+ BuildMI(*MI.getParent(), std::next(LastClauseInst), DebugLoc(),
+ TII->get(AMDGPU::KILL));
+ Kill.addUse(Reg, UseState | RegState::Kill, AMDGPU::NoSubRegister);
+
+ // Move the kill record from within the clause to the KILL instruction,
+ // keeping LiveVariables consistent with the modified MIR.
+ // findKill is guaranteed non-null: a kill flag within the clause
+ // implies LV recorded a kill for this register in this block.
+ MachineInstr *OldKill = LV->getVarInfo(Reg).findKill(&MBB);
+ assert(OldKill &&
+ "Kill flag in clause but no LV kill record in block?");
+ // replaceKillInstruction only updates the VarInfo::Kills list; clear
+ // the kill flag on the old instruction manually.
+ OldKill->clearRegisterKills(Reg, TRI);
+ LV->replaceKillInstruction(Reg, *OldKill, *Kill);
}
- // Update BlockPressure to reflect the committed clause. CurPressure
- // already has all clause defs accumulated. Subtract the uses that die
- // within the clause (those for which kills were inserted above).
+ // Update BlockPressure: CurPressure already has all clause defs
+ // accumulated; subtract the registers that died within the clause.
BlockPressure = CurPressure;
- for (auto &&R : Uses) {
- Register Reg = R.first;
- if (Reg.isPhysical())
- continue;
- if (!hasUseAfter(Reg, R.second.second, LastClauseInst, MBB))
- BlockPressure.inc(Reg, R.second.second, LaneBitmask::getNone(), *MRI);
- }
+ for (Register Reg : Kills)
+ BlockPressure.inc(Reg, MRI->getMaxLaneMaskForVReg(Reg),
+ LaneBitmask::getNone(), *MRI);
PressurePos = Next;
}
}
@@ -495,12 +454,14 @@ bool SSASIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
- return SSASIFormMemoryClausesImpl().run(MF);
+ LiveVariables &LV = getAnalysis<LiveVariablesWrapperPass>().getLV();
+ return SSASIFormMemoryClausesImpl().run(MF, LV);
}
PreservedAnalyses
SSASIFormMemoryClausesPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- SSASIFormMemoryClausesImpl().run(MF);
+ LiveVariables &LV = MFAM.getResult<LiveVariablesAnalysis>(MF);
+ SSASIFormMemoryClausesImpl().run(MF, LV);
return PreservedAnalyses::all();
}
diff --git a/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
new file mode 100644
index 0000000000000..71eb4e5550e1c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
@@ -0,0 +1,281 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack -run-pass=ssa-si-form-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=XNACK %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack -run-pass=ssa-si-form-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=NOXNACK %s
+
+# Verify basic behavior of the SSA-form memory clause pass. This pass is
+# similar to SIFormMemoryClauses, but runs before PHI elimination.
+
+# XNACK-LABEL: name: vector_clause
+# XNACK: %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+# XNACK-NEXT: %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+# XNACK-NEXT: %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 48, 0, implicit $exec
+# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: %5:vreg_64 = IMPLICIT_DEF
+
+# NOXNACK-LABEL: name: vector_clause
+# NOXNACK-NOT: KILL
+
+---
+name: vector_clause
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+ %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+ %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+ %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 48, 0, implicit $exec
+ %5:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORDX4 %5, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %5, %2, 16, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %5, %3, 32, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %5, %4, 48, 0, implicit $exec
+...
+
+# Pointer is live beyond the end of the clause (used by the stores), so no
+# KILL is needed.
+#
+# XNACK-LABEL: name: vector_clause_no_kill
+# XNACK: %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+# XNACK-NEXT: %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+# XNACK-NEXT: %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 48, 0, implicit $exec
+# XNACK-NEXT: GLOBAL_STORE_DWORDX4 %0
+
+---
+name: vector_clause_no_kill
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+ %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+ %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+ %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 48, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %0, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %0, %2, 16, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %0, %3, 32, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %0, %4, 48, 0, implicit $exec
+...
+
+# SMEM (scalar) clause: KILL inserted after the last load.
+#
+# XNACK-LABEL: name: scalar_clause
+# XNACK: %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0
+# XNACK-NEXT: %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0
+# XNACK-NEXT: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0
+# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: S_ENDPGM
+
+---
+name: scalar_clause
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+ %0:sreg_64 = COPY $sgpr4_sgpr5
+ %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0
+ %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0
+ S_ENDPGM 0, implicit %1, implicit %2, implicit %3
+...
+
+# A single load does not form a clause.
+#
+# XNACK-LABEL: name: single_load_no_clause
+# XNACK: %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+# XNACK-NOT: KILL
+
+---
+name: single_load_no_clause
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %0, %1, 0, 0, implicit $exec
+...
+
+# Subreg pointer: both sub-lane pairs die inside the clause (no post-clause use
+# of %0). LiveVars however only sets isKill() on the last whole-register use
+# (sub2_sub3 on the second load). We emit a whole-register KILL killed %0,
+# covering all lanes.
+#
+# XNACK-LABEL: name: subreg_ptr_killed_in_clause
+# XNACK: %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
+# XNACK-NEXT: %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
+# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: %3:vreg_64 = IMPLICIT_DEF
+#
+# NOXNACK-LABEL: name: subreg_ptr_killed_in_clause
+# NOXNACK-NOT: KILL
+
+---
+name: subreg_ptr_killed_in_clause
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_128 = IMPLICIT_DEF
+ %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
+ %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
+ %3:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORDX2 %3, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX2 %3, %2, 16, 0, implicit $exec
+...
+
+# Subreg pointer: sub0_sub1 is used after the clause (as store address), so
+# those lanes are live-out of the clause. Only sub2_sub3 dies inside the clause.
+# No KILL is needed because %0.sub2_sub3 does not have the killed flag given that
+# %0.sub0_sub1 (and thus %0) is still live after the last load.
+#
+# XNACK-LABEL: name: subreg_ptr_partial_live_after
+# XNACK: %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
+# XNACK-NEXT: %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
+# XNACK-NOT: KILL
+#
+# NOXNACK-LABEL: name: subreg_ptr_partial_live_after
+# NOXNACK-NOT: KILL
+
+---
+name: subreg_ptr_partial_live_after
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_128 = IMPLICIT_DEF
+ %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
+ %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
+ GLOBAL_STORE_DWORDX2 %0.sub0_sub1, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX2 %0.sub0_sub1, %2, 16, 0, implicit $exec
+...
+
+
+# SSA-specific: pointer register defined by a PHI is used as the clause
+# address. Here the PHI register %2 is used after the clause. So no KILL
+# is inserted.
+#
+# XNACK-LABEL: name: phi_pointer
+# XNACK: %2:vreg_64 = PHI %0, %bb.0, %1, %bb.1
+# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %2, 0, 0, implicit $exec
+# XNACK-NEXT: %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %2, 16, 0, implicit $exec
+# XNACK-NOT: KILL %2
+
+---
+name: phi_pointer
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1, %bb.2
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_64 = IMPLICIT_DEF
+ S_CBRANCH_EXECZ %bb.2, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+
+ bb.2:
+ %2:vreg_64 = PHI %0, %bb.0, %1, %bb.1
+ %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %2, 0, 0, implicit $exec
+ %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %2, 16, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %2, %3, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %2, %4, 16, 0, implicit $exec
+...
+
+# SSA-specific: pointer defined in a predecessor block, clause in successor.
+# The live range of the pointer must be extended past the last load in bb.1.
+#
+# XNACK-LABEL: name: pointer_from_predecessor
+# XNACK-LABEL: bb.1:
+# XNACK-NEXT: %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+# XNACK-NEXT: %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+# XNACK-NEXT: KILL killed %0
+
+---
+name: pointer_from_predecessor
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1
+ %0:vreg_64 = IMPLICIT_DEF
+
+ bb.1:
+ %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+ %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+ %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
+ %4:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORDX4 %4, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %4, %2, 16, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %4, %3, 32, 0, implicit $exec
+...
+
+# SSA-specific: verify the pass preserves SSA form (isSSA remains true).
+# We reuse the vector_clause input and grep the full MIR header fields.
+#
+# XNACK-LABEL: name: ssa_properties_preserved
+# XNACK: isSSA: true
+
+---
+name: ssa_properties_preserved
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+ %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
+ %3:vreg_64 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORDX4 %3, %1, 0, 0, implicit $exec
+ GLOBAL_STORE_DWORDX4 %3, %2, 16, 0, implicit $exec
+...
+
+# Clause length is capped at 15 instructions. 16 loads from %0 exceed the
+# limit so the clause stops at 15 and a KILL is inserted. The remaining load
+# from %0 and the two loads from %1 form a second clause.
+#
+# XNACK-LABEL: name: overflow_counter
+# XNACK: dead %2:vgpr_32 = GLOBAL_LOAD_DWORD %0, 0, 0, implicit $exec
+# XNACK-NEXT: dead %3:vgpr_32 = GLOBAL_LOAD_DWORD %0, 4, 0, implicit $exec
+# XNACK-NEXT: dead %4:vgpr_32 = GLOBAL_LOAD_DWORD %0, 8, 0, implicit $exec
+# XNACK-NEXT: dead %5:vgpr_32 = GLOBAL_LOAD_DWORD %0, 12, 0, implicit $exec
+# XNACK-NEXT: dead %6:vgpr_32 = GLOBAL_LOAD_DWORD %0, 16, 0, implicit $exec
+# XNACK-NEXT: dead %7:vgpr_32 = GLOBAL_LOAD_DWORD %0, 20, 0, implicit $exec
+# XNACK-NEXT: dead %8:vgpr_32 = GLOBAL_LOAD_DWORD %0, 24, 0, implicit $exec
+# XNACK-NEXT: dead %9:vgpr_32 = GLOBAL_LOAD_DWORD %0, 28, 0, implicit $exec
+# XNACK-NEXT: dead %10:vgpr_32 = GLOBAL_LOAD_DWORD %0, 32, 0, implicit $exec
+# XNACK-NEXT: dead %11:vgpr_32 = GLOBAL_LOAD_DWORD %0, 36, 0, implicit $exec
+# XNACK-NEXT: dead %12:vgpr_32 = GLOBAL_LOAD_DWORD %0, 40, 0, implicit $exec
+# XNACK-NEXT: dead %13:vgpr_32 = GLOBAL_LOAD_DWORD %0, 44, 0, implicit $exec
+# XNACK-NEXT: dead %14:vgpr_32 = GLOBAL_LOAD_DWORD %0, 48, 0, implicit $exec
+# XNACK-NEXT: dead %15:vgpr_32 = GLOBAL_LOAD_DWORD %0, 52, 0, implicit $exec
+# XNACK-NEXT: dead %16:vgpr_32 = GLOBAL_LOAD_DWORD %0, 56, 0, implicit $exec
+# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: dead %17:vgpr_32 = GLOBAL_LOAD_DWORD %1, 60, 0, implicit $exec
+# XNACK-NEXT: dead %18:vgpr_32 = GLOBAL_LOAD_DWORD %1, 64, 0, implicit $exec
+# XNACK-NEXT: KILL killed %1
+
+---
+name: overflow_counter
+tracksRegLiveness: true
+body: |
+ bb.0:
+ %0:vreg_64 = IMPLICIT_DEF
+ %1:vreg_64 = IMPLICIT_DEF
+ dead %2:vgpr_32 = GLOBAL_LOAD_DWORD %0, 0, 0, implicit $exec
+ dead %3:vgpr_32 = GLOBAL_LOAD_DWORD %0, 4, 0, implicit $exec
+ dead %4:vgpr_32 = GLOBAL_LOAD_DWORD %0, 8, 0, implicit $exec
+ dead %5:vgpr_32 = GLOBAL_LOAD_DWORD %0, 12, 0, implicit $exec
+ dead %6:vgpr_32 = GLOBAL_LOAD_DWORD %0, 16, 0, implicit $exec
+ dead %7:vgpr_32 = GLOBAL_LOAD_DWORD %0, 20, 0, implicit $exec
+ dead %8:vgpr_32 = GLOBAL_LOAD_DWORD %0, 24, 0, implicit $exec
+ dead %9:vgpr_32 = GLOBAL_LOAD_DWORD %0, 28, 0, implicit $exec
+ dead %10:vgpr_32 = GLOBAL_LOAD_DWORD %0, 32, 0, implicit $exec
+ dead %11:vgpr_32 = GLOBAL_LOAD_DWORD %0, 36, 0, implicit $exec
+ dead %12:vgpr_32 = GLOBAL_LOAD_DWORD %0, 40, 0, implicit $exec
+ dead %13:vgpr_32 = GLOBAL_LOAD_DWORD %0, 44, 0, implicit $exec
+ dead %14:vgpr_32 = GLOBAL_LOAD_DWORD %0, 48, 0, implicit $exec
+ dead %15:vgpr_32 = GLOBAL_LOAD_DWORD %0, 52, 0, implicit $exec
+ dead %16:vgpr_32 = GLOBAL_LOAD_DWORD %0, 56, 0, implicit $exec
+ dead %17:vgpr_32 = GLOBAL_LOAD_DWORD %1, 60, 0, implicit $exec
+ dead %18:vgpr_32 = GLOBAL_LOAD_DWORD %1, 64, 0, implicit $exec
+...
+
>From d2891ece559791b6889f6fc6f206c372c3a76f6f Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86 at yahoo.com>
Date: Mon, 27 Jul 2026 14:36:13 -0700
Subject: [PATCH 4/5] Revert to using LiveIntervals
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 +-
.../Target/AMDGPU/SSASIFormMemoryClauses.cpp | 263 ++++++++----------
.../test/CodeGen/AMDGPU/ssa-memory-clause.mir | 24 +-
3 files changed, 131 insertions(+), 165 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d3bce6a2f1764..df8dc27f9699c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1770,12 +1770,11 @@ void GCNPassConfig::addOptimizedRegAlloc() {
insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
// Run the SSA form of the memory clause pass before PHI elimination.
- // LiveVariables is the anchor: it runs in SSA form and sets kill flags that
- // our pass relies on for intra-block liveness tracking.
- // TODO: Once PR #161054 (SSAMachineScheduler) is merged, anchor this pass
- // after SSAMachineSchedulerID instead of LiveVariablesID.
+ // MachineLoopInfo is the last pass before PHIElimination in the base
+ // pipeline, so this places the pass as late as possible while the function
+ // is still in SSA form.
if (EnableSSASIFormMemoryClauses)
- insertPass(&LiveVariablesID, &SSASIFormMemoryClausesID);
+ insertPass(&MachineLoopInfoID, &SSASIFormMemoryClausesID);
TargetPassConfig::addOptimizedRegAlloc();
}
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
index c52422a180f63..6b6e6e318c6ca 100644
--- a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
@@ -12,17 +12,13 @@
/// is enabled, preventing a load from overwriting a pointer and requiring a
/// soft clause break.
///
-/// TODO: Once PR #161054 (SSAMachineScheduler) is merged this pass should be
-/// placed immediately after SSAMachineScheduler in the pipeline.
-///
//===----------------------------------------------------------------------===//
#include "SSASIFormMemoryClauses.h"
#include "AMDGPU.h"
#include "GCNRegPressure.h"
#include "SIMachineFunctionInfo.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/InitializePasses.h"
using namespace llvm;
@@ -43,23 +39,25 @@ class SSASIFormMemoryClausesImpl {
bool canBundle(const MachineInstr &MI, const RegUse &Defs,
const RegUse &Uses) const;
- bool checkPressure(const MachineInstr &MI, GCNRegPressure &CurPressure);
- void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
+ bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
+ void collectRegUses(const MachineInstr &MI, RegUse &Defs,
+ RegUse &Uses) const;
bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
- GCNRegPressure &CurPressure);
+ GCNDownwardRPTracker &RPT);
const GCNSubtarget *ST;
const SIRegisterInfo *TRI;
const MachineRegisterInfo *MRI;
SIMachineFunctionInfo *MFI;
- LiveVariables *LV;
+ LiveIntervals *LIS;
unsigned LastRecordedOccupancy;
unsigned MaxVGPRs;
unsigned MaxSGPRs;
public:
- bool run(MachineFunction &MF, LiveVariables &LV);
+ SSASIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
+ bool run(MachineFunction &MF);
};
class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
@@ -75,7 +73,7 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<LiveVariablesWrapperPass>();
+ AU.addRequired<LiveIntervalsWrapperPass>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -88,7 +86,7 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
INITIALIZE_PASS_BEGIN(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
-INITIALIZE_PASS_DEPENDENCY(LiveVariablesWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
INITIALIZE_PASS_END(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
"SSA SI Form memory clauses", false, false)
@@ -192,28 +190,15 @@ bool SSASIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
// Since all defs in the clause are early clobber we can run out of registers.
// Function returns false if pressure would hit the limit if instruction is
// bundled into a memory clause.
-//
-// We accumulate pressure monotonically across the clause: because all defs are
-// marked early-clobber they remain live until the clause end, so we never
-// subtract pressure for uses that die mid-clause. This is conservative and
-// avoids the need for LiveIntervals.
bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
- GCNRegPressure &CurPressure) {
- // Speculatively add this instruction's virtual defs to the running pressure.
- // Physical register defs are skipped: they are not allocatable slots and
- // GCNRegPressure::inc() requires a virtual register.
- GCNRegPressure NewPressure = CurPressure;
- for (const MachineOperand &MO : MI.defs()) {
- if (!MO.isReg() || !MO.getReg().isVirtual())
- continue;
- Register Reg = MO.getReg();
- LaneBitmask Mask = MO.getSubReg()
- ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
- : MRI->getMaxLaneMaskForVReg(Reg);
- NewPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
- }
-
- unsigned Occupancy = NewPressure.getOccupancy(
+ GCNDownwardRPTracker &RPT) {
+ // NB: skip advanceBeforeNext() call. Since all defs will be marked
+ // early-clobber they will all stay alive at least to the end of the
+ // clause. Therefore we should not decrease pressure even if a load
+ // pointer becomes dead and could otherwise be reused for destination.
+ RPT.advanceToNext();
+ GCNRegPressure MaxPressure = RPT.moveMaxPressure();
+ unsigned Occupancy = MaxPressure.getOccupancy(
*ST,
MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
@@ -227,10 +212,9 @@ bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
// tracking does not account for the alignment requirements for SGPRs, or the
// fragmentation of registers the allocator will need to satisfy.
if (Occupancy >= MFI->getMinAllowedOccupancy() &&
- NewPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
- NewPressure.getSGPRNum() <= MaxSGPRs / 2) {
+ MaxPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
+ MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
LastRecordedOccupancy = Occupancy;
- CurPressure = NewPressure;
return true;
}
return false;
@@ -266,18 +250,18 @@ void SSASIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
// def/use maps are not updated.
bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
RegUse &Defs, RegUse &Uses,
- GCNRegPressure &CurPressure) {
+ GCNDownwardRPTracker &RPT) {
if (!canBundle(MI, Defs, Uses))
return false;
- if (!checkPressure(MI, CurPressure))
+ if (!checkPressure(MI, RPT))
return false;
collectRegUses(MI, Defs, Uses);
return true;
}
-bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
+bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
if (!ST->isXNACKEnabled())
return false;
@@ -286,7 +270,7 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
TRI = ST->getRegisterInfo();
MRI = &MF.getRegInfo();
MFI = MF.getInfo<SIMachineFunctionInfo>();
- LV = &LVIn;
+ SlotIndexes *Ind = LIS->getSlotIndexes();
bool Changed = false;
MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
@@ -295,45 +279,7 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
"amdgpu-max-memory-clause", SSAMaxClause);
for (MachineBasicBlock &MBB : MF) {
- // BlockPressure tracks the register pressure at the current scan position
- // within MBB. It is seeded with virtual registers live-in to this block
- // (as computed by LiveVariables), then updated instruction-by-instruction:
- // virtual register defs increase pressure; uses with kill flags decrease
- // it. In SSA form, kill flags are reliable (each vreg has exactly one
- // def), so this gives accurate intra-block liveness.
- GCNRegPressure BlockPressure;
- for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
- Register Reg = Register::index2VirtReg(I);
- if (LV->isLiveIn(Reg, MBB)) {
- LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(Reg);
- BlockPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
- }
- }
-
- // PressurePos is the next instruction to be consumed into BlockPressure.
- // It may lag behind the outer loop iterator when the inner clause-extension
- // loop advances Next past instructions not admitted to a clause.
- // advanceBlockPressure() catches it up before each clause attempt.
- auto PressurePos = MBB.instr_begin();
-
- auto advanceBlockPressure = [&](MachineBasicBlock::instr_iterator Target) {
- while (PressurePos != Target) {
- const MachineInstr &CurMI = *PressurePos++;
- if (CurMI.isMetaInstruction())
- continue;
- for (const MachineOperand &MO : CurMI.operands()) {
- if (!MO.isReg() || !MO.getReg().isVirtual())
- continue;
- Register Reg = MO.getReg();
- LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
- if (MO.isDef())
- BlockPressure.inc(Reg, LaneBitmask::getNone(), Mask, *MRI);
- else if (MO.isKill())
- BlockPressure.inc(Reg, Mask, LaneBitmask::getNone(), *MRI);
- }
- }
- };
-
+ GCNDownwardRPTracker RPT(*LIS);
MachineBasicBlock::instr_iterator Next;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
MachineInstr &MI = *I;
@@ -344,40 +290,24 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
bool IsVMEM = isVMEMClauseInst(MI);
- if (!isValidClauseInst(MI, IsVMEM)) {
- advanceBlockPressure(Next);
+ if (!isValidClauseInst(MI, IsVMEM))
continue;
- }
- // Bring BlockPressure up to (but not including) MI, then snapshot it as
- // the baseline pressure entering this potential clause.
- advanceBlockPressure(I);
- GCNRegPressure CurPressure = BlockPressure;
+ if (!RPT.getNext().isValid())
+ RPT.reset(MI, MBB.end());
+ else { // Advance the state to the current MI.
+ RPT.advance(MachineBasicBlock::const_iterator(MI));
+ RPT.advanceBeforeNext();
+ }
+ const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
RegUse Defs, Uses;
- // Kills: virtual registers with isKill() on any use inside the clause.
- // These registers die within the clause and need a whole-register KILL
- // pseudo after the last load to extend their live range past the
- // early-clobber defs. The specific subreg that LV flagged does not
- // matter; we always emit a whole-register KILL.
- DenseSet<Register> Kills;
-
- auto collectKills = [&](const MachineInstr &Instr) {
- for (const MachineOperand &MO : Instr.operands()) {
- if (!MO.isReg() || MO.isDef() || !MO.isKill() ||
- !MO.getReg().isVirtual())
- continue;
- Kills.insert(MO.getReg());
- }
- };
-
- if (!processRegUses(MI, Defs, Uses, CurPressure)) {
- advanceBlockPressure(Next);
+ if (!processRegUses(MI, Defs, Uses, RPT)) {
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
continue;
}
- collectKills(MI);
- MachineBasicBlock::instr_iterator LastClauseInst = Next;
+ MachineBasicBlock::iterator LastClauseInst = Next;
unsigned Length = 1;
for (; Next != E && Length < FuncMaxClause; ++Next) {
// Debug instructions should not change the kill insertion.
@@ -390,18 +320,14 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
// A load from pointer which was loaded inside the same bundle is an
// impossible clause because we will need to write and read the same
// register inside. In this case processRegUses will return false.
- if (!processRegUses(*Next, Defs, Uses, CurPressure))
+ if (!processRegUses(*Next, Defs, Uses, RPT))
break;
- collectKills(*Next);
LastClauseInst = Next;
++Length;
}
if (Length < 2) {
- // Clause did not form; process MI normally. Instructions examined by
- // the inner loop but not admitted will be caught up by
- // advanceBlockPressure() at the start of the next outer iteration.
- advanceBlockPressure(std::next(I));
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
continue;
}
@@ -410,40 +336,81 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF, LiveVariables &LVIn) {
assert(!LastClauseInst->isMetaInstruction());
- // For each register killed within the clause, insert a whole-register
- // KILL pseudo after the clause to extend its liveness through the
- // early-clobber defs. Registers not in Kills are live past the clause
- // and need nothing.
- for (Register Reg : Kills) {
- auto UseIt = Uses.find(Reg);
- assert(UseIt != Uses.end());
- RegState UseState = UseIt->second.first & ~RegState::Kill;
-
- MachineInstrBuilder Kill =
- BuildMI(*MI.getParent(), std::next(LastClauseInst), DebugLoc(),
- TII->get(AMDGPU::KILL));
- Kill.addUse(Reg, UseState | RegState::Kill, AMDGPU::NoSubRegister);
-
- // Move the kill record from within the clause to the KILL instruction,
- // keeping LiveVariables consistent with the modified MIR.
- // findKill is guaranteed non-null: a kill flag within the clause
- // implies LV recorded a kill for this register in this block.
- MachineInstr *OldKill = LV->getVarInfo(Reg).findKill(&MBB);
- assert(OldKill &&
- "Kill flag in clause but no LV kill record in block?");
- // replaceKillInstruction only updates the VarInfo::Kills list; clear
- // the kill flag on the old instruction manually.
- OldKill->clearRegisterKills(Reg, TRI);
- LV->replaceKillInstruction(Reg, *OldKill, *Kill);
+ SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(MI);
+ SlotIndex ClauseLiveOutIdx =
+ LIS->getInstructionIndex(*LastClauseInst).getNextIndex();
+
+ // Track the last inserted kill.
+ MachineInstrBuilder Kill;
+
+ // Insert one kill per register, with operands covering all necessary
+ // subregisters.
+ for (auto &&R : Uses) {
+ Register Reg = R.first;
+ if (Reg.isPhysical())
+ continue;
+
+ // Collect the register operands we should extend the live ranges of.
+ SmallVector<std::tuple<RegState, unsigned>> KillOps;
+ const LiveInterval &LI = LIS->getInterval(R.first);
+
+ if (!LI.hasSubRanges()) {
+ if (!LI.liveAt(ClauseLiveOutIdx)) {
+ KillOps.emplace_back(R.second.first | RegState::Kill,
+ AMDGPU::NoSubRegister);
+ }
+ } else {
+ LaneBitmask KilledMask;
+ for (const LiveInterval::SubRange &SR : LI.subranges()) {
+ if (SR.liveAt(ClauseLiveInIdx) && !SR.liveAt(ClauseLiveOutIdx))
+ KilledMask |= SR.LaneMask;
+ }
+
+ if (KilledMask.none())
+ continue;
+
+ SmallVector<unsigned> KilledIndexes;
+ bool Success = TRI->getCoveringSubRegIndexes(
+ MRI->getRegClass(Reg), KilledMask, KilledIndexes);
+ (void)Success;
+ assert(Success && "Failed to find subregister mask to cover lanes");
+ for (unsigned SubReg : KilledIndexes) {
+ KillOps.emplace_back(R.second.first | RegState::Kill, SubReg);
+ }
+ }
+
+ if (KillOps.empty())
+ continue;
+
+ Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst), DebugLoc(),
+ TII->get(AMDGPU::KILL));
+ for (auto &Op : KillOps)
+ Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
+ Ind->insertMachineInstrInMaps(*Kill);
}
- // Update BlockPressure: CurPressure already has all clause defs
- // accumulated; subtract the registers that died within the clause.
- BlockPressure = CurPressure;
- for (Register Reg : Kills)
- BlockPressure.inc(Reg, MRI->getMaxLaneMaskForVReg(Reg),
- LaneBitmask::getNone(), *MRI);
- PressurePos = Next;
+ // Restore the state after processing the end of the bundle.
+ RPT.reset(MI, MBB.end(), &LiveRegsCopy);
+
+ if (!Kill)
+ continue;
+
+ for (auto &&R : Defs) {
+ Register Reg = R.first;
+ Uses.erase(Reg);
+ if (Reg.isPhysical())
+ continue;
+ LIS->removeInterval(Reg);
+ LIS->createAndComputeVirtRegInterval(Reg);
+ }
+
+ for (auto &&R : Uses) {
+ Register Reg = R.first;
+ if (Reg.isPhysical())
+ continue;
+ LIS->removeInterval(Reg);
+ LIS->createAndComputeVirtRegInterval(Reg);
+ }
}
}
@@ -454,14 +421,14 @@ bool SSASIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
- LiveVariables &LV = getAnalysis<LiveVariablesWrapperPass>().getLV();
- return SSASIFormMemoryClausesImpl().run(MF, LV);
+ LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+ return SSASIFormMemoryClausesImpl(LIS).run(MF);
}
PreservedAnalyses
SSASIFormMemoryClausesPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
- LiveVariables &LV = MFAM.getResult<LiveVariablesAnalysis>(MF);
- SSASIFormMemoryClausesImpl().run(MF, LV);
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ SSASIFormMemoryClausesImpl(&LIS).run(MF);
return PreservedAnalyses::all();
}
diff --git a/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
index 71eb4e5550e1c..b5bb87ade1684 100644
--- a/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
+++ b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
@@ -9,7 +9,7 @@
# XNACK-NEXT: %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
# XNACK-NEXT: %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 48, 0, implicit $exec
-# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: KILL %0
# XNACK-NEXT: %5:vreg_64 = IMPLICIT_DEF
# NOXNACK-LABEL: name: vector_clause
@@ -64,7 +64,7 @@ body: |
# XNACK: %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0
# XNACK-NEXT: %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0
# XNACK-NEXT: %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0
-# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: KILL %0
# XNACK-NEXT: S_ENDPGM
---
@@ -97,14 +97,13 @@ body: |
...
# Subreg pointer: both sub-lane pairs die inside the clause (no post-clause use
-# of %0). LiveVars however only sets isKill() on the last whole-register use
-# (sub2_sub3 on the second load). We emit a whole-register KILL killed %0,
-# covering all lanes.
+# of %0). LiveIntervals detects no live lanes past the clause end, so we emit
+# a whole-register KILL %0, covering all lanes.
#
# XNACK-LABEL: name: subreg_ptr_killed_in_clause
# XNACK: %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
# XNACK-NEXT: %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
-# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: KILL %0
# XNACK-NEXT: %3:vreg_64 = IMPLICIT_DEF
#
# NOXNACK-LABEL: name: subreg_ptr_killed_in_clause
@@ -125,13 +124,14 @@ body: |
# Subreg pointer: sub0_sub1 is used after the clause (as store address), so
# those lanes are live-out of the clause. Only sub2_sub3 dies inside the clause.
-# No KILL is needed because %0.sub2_sub3 does not have the killed flag given that
-# %0.sub0_sub1 (and thus %0) is still live after the last load.
+# LiveIntervals detects the sub2_sub3 subrange dies at the clause end, so a
+# KILL for that subrange is emitted to extend its liveness through the
+# early-clobber defs.
#
# XNACK-LABEL: name: subreg_ptr_partial_live_after
# XNACK: %1:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub0_sub1, 0, 0, implicit $exec
# XNACK-NEXT: %2:vreg_64 = GLOBAL_LOAD_DWORDX2 %0.sub2_sub3, 16, 0, implicit $exec
-# XNACK-NOT: KILL
+# XNACK-NEXT: KILL %0.sub2_sub3
#
# NOXNACK-LABEL: name: subreg_ptr_partial_live_after
# NOXNACK-NOT: KILL
@@ -189,7 +189,7 @@ body: |
# XNACK-NEXT: %1:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
# XNACK-NEXT: %2:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 16, 0, implicit $exec
# XNACK-NEXT: %3:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 32, 0, implicit $exec
-# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: KILL %0
---
name: pointer_from_predecessor
@@ -248,10 +248,10 @@ body: |
# XNACK-NEXT: dead %14:vgpr_32 = GLOBAL_LOAD_DWORD %0, 48, 0, implicit $exec
# XNACK-NEXT: dead %15:vgpr_32 = GLOBAL_LOAD_DWORD %0, 52, 0, implicit $exec
# XNACK-NEXT: dead %16:vgpr_32 = GLOBAL_LOAD_DWORD %0, 56, 0, implicit $exec
-# XNACK-NEXT: KILL killed %0
+# XNACK-NEXT: KILL %0
# XNACK-NEXT: dead %17:vgpr_32 = GLOBAL_LOAD_DWORD %1, 60, 0, implicit $exec
# XNACK-NEXT: dead %18:vgpr_32 = GLOBAL_LOAD_DWORD %1, 64, 0, implicit $exec
-# XNACK-NEXT: KILL killed %1
+# XNACK-NEXT: KILL %1
---
name: overflow_counter
>From dd6fe1af331bcbb60f08d7141e4cbb8b0c8e3948 Mon Sep 17 00:00:00 2001
From: Jun Wang <jwang86 at yahoo.com>
Date: Mon, 27 Jul 2026 15:22:19 -0700
Subject: [PATCH 5/5] Change name to AMDGPUFormSSAMemoryClauses
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 6 +--
...ses.cpp => AMDGPUFormSSAMemoryClauses.cpp} | 50 +++++++++----------
...Clauses.h => AMDGPUFormSSAMemoryClauses.h} | 12 ++---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 12 ++---
llvm/lib/Target/AMDGPU/CMakeLists.txt | 2 +-
.../test/CodeGen/AMDGPU/ssa-memory-clause.mir | 4 +-
6 files changed, 43 insertions(+), 43 deletions(-)
rename llvm/lib/Target/AMDGPU/{SSASIFormMemoryClauses.cpp => AMDGPUFormSSAMemoryClauses.cpp} (89%)
rename llvm/lib/Target/AMDGPU/{SSASIFormMemoryClauses.h => AMDGPUFormSSAMemoryClauses.h} (59%)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index bf52cd32b1a34..8c4a2980122ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -55,7 +55,7 @@ FunctionPass *createSIMemoryLegalizerPass();
FunctionPass *createSIInsertWaitcntsPass();
FunctionPass *createSIPreAllocateWWMRegsLegacyPass();
FunctionPass *createSIFormMemoryClausesLegacyPass();
-FunctionPass *createSSASIFormMemoryClausesLegacyPass();
+FunctionPass *createAMDGPUFormSSAMemoryClausesLegacyPass();
FunctionPass *createSIPostRABundlerPass();
FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
@@ -552,8 +552,8 @@ extern char &SIInsertWaitcntsID;
void initializeSIFormMemoryClausesLegacyPass(PassRegistry &);
extern char &SIFormMemoryClausesID;
-void initializeSSASIFormMemoryClausesLegacyPass(PassRegistry &);
-extern char &SSASIFormMemoryClausesID;
+void initializeAMDGPUFormSSAMemoryClausesLegacyPass(PassRegistry &);
+extern char &AMDGPUFormSSAMemoryClausesID;
void initializeSIPostRABundlerLegacyPass(PassRegistry &);
extern char &SIPostRABundlerLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.cpp
similarity index 89%
rename from llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
rename to llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.cpp
index 6b6e6e318c6ca..cd1b806144fd7 100644
--- a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.cpp
@@ -1,4 +1,4 @@
-//===-- SSASIFormMemoryClauses.cpp ----------------------------------------===//
+//===-- AMDGPUFormSSAMemoryClauses.cpp ------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -14,7 +14,7 @@
///
//===----------------------------------------------------------------------===//
-#include "SSASIFormMemoryClauses.h"
+#include "AMDGPUFormSSAMemoryClauses.h"
#include "AMDGPU.h"
#include "GCNRegPressure.h"
#include "SIMachineFunctionInfo.h"
@@ -23,7 +23,7 @@
using namespace llvm;
-#define DEBUG_TYPE "ssa-si-form-memory-clauses"
+#define DEBUG_TYPE "amdgpu-form-ssa-memory-clauses"
// Clauses longer then 15 instructions would overflow one of the counters
// and stall. They can stall even earlier if there are outstanding counters.
@@ -34,7 +34,7 @@ static cl::opt<unsigned> SSAMaxClause(
namespace {
-class SSASIFormMemoryClausesImpl {
+class AMDGPUFormSSAMemoryClausesImpl {
using RegUse = DenseMap<unsigned, std::pair<RegState, LaneBitmask>>;
bool canBundle(const MachineInstr &MI, const RegUse &Defs,
@@ -56,20 +56,20 @@ class SSASIFormMemoryClausesImpl {
unsigned MaxSGPRs;
public:
- SSASIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
+ AMDGPUFormSSAMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
bool run(MachineFunction &MF);
};
-class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
+class AMDGPUFormSSAMemoryClausesLegacy : public MachineFunctionPass {
public:
static char ID;
- SSASIFormMemoryClausesLegacy() : MachineFunctionPass(ID) {}
+ AMDGPUFormSSAMemoryClausesLegacy() : MachineFunctionPass(ID) {}
bool runOnMachineFunction(MachineFunction &MF) override;
StringRef getPassName() const override {
- return "SSA SI Form memory clauses";
+ return "AMDGPU Form SSA Memory Clauses";
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -84,18 +84,18 @@ class SSASIFormMemoryClausesLegacy : public MachineFunctionPass {
} // End anonymous namespace.
-INITIALIZE_PASS_BEGIN(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
- "SSA SI Form memory clauses", false, false)
+INITIALIZE_PASS_BEGIN(AMDGPUFormSSAMemoryClausesLegacy, DEBUG_TYPE,
+ "AMDGPU Form SSA Memory Clauses", false, false)
INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
-INITIALIZE_PASS_END(SSASIFormMemoryClausesLegacy, DEBUG_TYPE,
- "SSA SI Form memory clauses", false, false)
+INITIALIZE_PASS_END(AMDGPUFormSSAMemoryClausesLegacy, DEBUG_TYPE,
+ "AMDGPU Form SSA Memory Clauses", false, false)
-char SSASIFormMemoryClausesLegacy::ID = 0;
+char AMDGPUFormSSAMemoryClausesLegacy::ID = 0;
-char &llvm::SSASIFormMemoryClausesID = SSASIFormMemoryClausesLegacy::ID;
+char &llvm::AMDGPUFormSSAMemoryClausesID = AMDGPUFormSSAMemoryClausesLegacy::ID;
-FunctionPass *llvm::createSSASIFormMemoryClausesLegacyPass() {
- return new SSASIFormMemoryClausesLegacy();
+FunctionPass *llvm::createAMDGPUFormSSAMemoryClausesLegacyPass() {
+ return new AMDGPUFormSSAMemoryClausesLegacy();
}
static bool isVMEMClauseInst(const MachineInstr &MI) {
@@ -152,7 +152,7 @@ static RegState getMopState(const MachineOperand &MO) {
// Returns false if there is a use of a def already in the map.
// In this case we must break the clause.
-bool SSASIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
+bool AMDGPUFormSSAMemoryClausesImpl::canBundle(const MachineInstr &MI,
const RegUse &Defs,
const RegUse &Uses) const {
// Check interference with defs.
@@ -190,7 +190,7 @@ bool SSASIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
// Since all defs in the clause are early clobber we can run out of registers.
// Function returns false if pressure would hit the limit if instruction is
// bundled into a memory clause.
-bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
+bool AMDGPUFormSSAMemoryClausesImpl::checkPressure(const MachineInstr &MI,
GCNDownwardRPTracker &RPT) {
// NB: skip advanceBeforeNext() call. Since all defs will be marked
// early-clobber they will all stay alive at least to the end of the
@@ -221,7 +221,7 @@ bool SSASIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
}
// Collect register defs and uses along with their lane masks and states.
-void SSASIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
+void AMDGPUFormSSAMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
RegUse &Defs,
RegUse &Uses) const {
for (const MachineOperand &MO : MI.operands()) {
@@ -248,7 +248,7 @@ void SSASIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
// Check register def/use conflicts, occupancy limits and collect def/use maps.
// Return true if instruction can be bundled with previous. If it cannot
// def/use maps are not updated.
-bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
+bool AMDGPUFormSSAMemoryClausesImpl::processRegUses(const MachineInstr &MI,
RegUse &Defs, RegUse &Uses,
GCNDownwardRPTracker &RPT) {
if (!canBundle(MI, Defs, Uses))
@@ -261,7 +261,7 @@ bool SSASIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
return true;
}
-bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
+bool AMDGPUFormSSAMemoryClausesImpl::run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
if (!ST->isXNACKEnabled())
return false;
@@ -417,18 +417,18 @@ bool SSASIFormMemoryClausesImpl::run(MachineFunction &MF) {
return Changed;
}
-bool SSASIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
+bool AMDGPUFormSSAMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
- return SSASIFormMemoryClausesImpl(LIS).run(MF);
+ return AMDGPUFormSSAMemoryClausesImpl(LIS).run(MF);
}
PreservedAnalyses
-SSASIFormMemoryClausesPass::run(MachineFunction &MF,
+AMDGPUFormSSAMemoryClausesPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
- SSASIFormMemoryClausesImpl(&LIS).run(MF);
+ AMDGPUFormSSAMemoryClausesImpl(&LIS).run(MF);
return PreservedAnalyses::all();
}
diff --git a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h b/llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.h
similarity index 59%
rename from llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h
rename to llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.h
index 099b975623225..29c6de60065b5 100644
--- a/llvm/lib/Target/AMDGPU/SSASIFormMemoryClauses.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUFormSSAMemoryClauses.h
@@ -1,4 +1,4 @@
-//===- SSASIFormMemoryClauses.h ---------------------------------*- C++ -*-===//
+//===- AMDGPUFormSSAMemoryClauses.h -----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,18 +6,18 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
-#define LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUFORMSSAMEMORYCLASSES_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUFORMSSAMEMORYCLASSES_H
#include "llvm/CodeGen/MachinePassManager.h"
namespace llvm {
-class SSASIFormMemoryClausesPass
- : public OptionalPassInfoMixin<SSASIFormMemoryClausesPass> {
+class AMDGPUFormSSAMemoryClausesPass
+ : public OptionalPassInfoMixin<AMDGPUFormSSAMemoryClausesPass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
};
} // namespace llvm
-#endif // LLVM_LIB_TARGET_AMDGPU_SSASIFORMMEMORYCLAUSES_H
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUFORMSSAMEMORYCLASSES_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index df8dc27f9699c..8f74befefd882 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -22,6 +22,7 @@
#include "AMDGPUCtorDtorLowering.h"
#include "AMDGPUExportClustering.h"
#include "AMDGPUExportKernelRuntimeHandles.h"
+#include "AMDGPUFormSSAMemoryClauses.h"
#include "AMDGPUHazardLatency.h"
#include "AMDGPUIGroupLP.h"
#include "AMDGPUISelDAGToDAG.h"
@@ -67,7 +68,6 @@
#include "SIPreAllocateWWMRegs.h"
#include "SIShrinkInstructions.h"
#include "SIWholeQuadMode.h"
-#include "SSASIFormMemoryClauses.h"
#include "TargetInfo/AMDGPUTargetInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/Analysis/CGSCCPassManager.h"
@@ -565,7 +565,7 @@ static cl::opt<bool> EnablePreRAOptimizations(
cl::desc("Enable Pre-RA optimizations pass"), cl::init(true),
cl::Hidden);
-static cl::opt<bool> EnableSSASIFormMemoryClauses(
+static cl::opt<bool> EnableAMDGPUFormSSAMemoryClauses(
"amdgpu-enable-ssa-form-memory-clauses",
cl::desc("Enable SSA form memory clause pass (before PHI elimination)"),
cl::init(false), cl::Hidden);
@@ -720,7 +720,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSIOptimizeExecMaskingLegacyPass(*PR);
initializeSIPreAllocateWWMRegsLegacyPass(*PR);
initializeSIFormMemoryClausesLegacyPass(*PR);
- initializeSSASIFormMemoryClausesLegacyPass(*PR);
+ initializeAMDGPUFormSSAMemoryClausesLegacyPass(*PR);
initializeSIPostRABundlerLegacyPass(*PR);
initializeGCNCreateVOPDLegacyPass(*PR);
initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
@@ -1766,15 +1766,15 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// This is not an essential optimization and it has a noticeable impact on
// compilation time, so we only enable it from O2.
if (TM->getOptLevel() > CodeGenOptLevel::Less &&
- !EnableSSASIFormMemoryClauses)
+ !EnableAMDGPUFormSSAMemoryClauses)
insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
// Run the SSA form of the memory clause pass before PHI elimination.
// MachineLoopInfo is the last pass before PHIElimination in the base
// pipeline, so this places the pass as late as possible while the function
// is still in SSA form.
- if (EnableSSASIFormMemoryClauses)
- insertPass(&MachineLoopInfoID, &SSASIFormMemoryClausesID);
+ if (EnableAMDGPUFormSSAMemoryClauses)
+ insertPass(&MachineLoopInfoID, &AMDGPUFormSSAMemoryClausesID);
TargetPassConfig::addOptimizedRegAlloc();
}
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 8273851731b17..e6b58717a52cd 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -163,7 +163,7 @@ add_llvm_target(AMDGPUCodeGen
SIFixVGPRCopies.cpp
SIFoldOperands.cpp
SIFormMemoryClauses.cpp
- SSASIFormMemoryClauses.cpp
+ AMDGPUFormSSAMemoryClauses.cpp
SIFrameLowering.cpp
SIInsertHardClauses.cpp
SIInsertWaitcnts.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
index b5bb87ade1684..b333026a1e380 100644
--- a/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
+++ b/llvm/test/CodeGen/AMDGPU/ssa-memory-clause.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack -run-pass=ssa-si-form-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=XNACK %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack -run-pass=ssa-si-form-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=NOXNACK %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=+xnack -run-pass=amdgpu-form-ssa-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=XNACK %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -mattr=-xnack -run-pass=amdgpu-form-ssa-memory-clauses -verify-machineinstrs -simplify-mir -o - %s | FileCheck -check-prefix=NOXNACK %s
# Verify basic behavior of the SSA-form memory clause pass. This pass is
# similar to SIFormMemoryClauses, but runs before PHI elimination.
More information about the llvm-commits
mailing list