[llvm-branch-commits] [llvm] Allocate and reserve registers for address space 13 objects (PR #214248)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 5 08:12:59 PDT 2026
https://github.com/doru1004 created https://github.com/llvm/llvm-project/pull/214248
Allocate and reserve registers for address space 13 objects
>From 42dbd04ed1152c5a634bced18f905ba7cf2e1474 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 5 Aug 2026 17:57:28 +0300
Subject: [PATCH] Allocate and reserve registers for address space 13 objects
---
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 9 +
llvm/lib/Target/AMDGPU/AMDGPU.h | 23 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 34 +-
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 2 +
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 19 +
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 26 ++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 19 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 2 +
.../AMDGPU/AMDGPUPrivateObjectVGPRs.cpp | 339 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 152 +++++++-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 23 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 57 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +-
llvm/lib/Target/AMDGPU/SIInstructions.td | 19 +
.../AddressSpaceVGPR/as-vgpr-across-call.ll | 70 ++++
.../AddressSpaceVGPR/as-vgpr-allocate.ll | 133 +++++++
.../as-vgpr-object-regalloc.ll | 178 +++++++++
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 4 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 9 +-
22 files changed, 1121 insertions(+), 21 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 21882e247c027..1048f99752643 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -4292,4 +4292,13 @@ def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic<
/// incoming block.
def int_amdgcn_dead: DefaultAttrsIntrinsic<[llvm_any_ty], [],
[IntrNoMem]>;
+
+/// Backend-only intrinsics analogous to llvm.lifetime.{start,end} for marking
+/// the lifetimes of allocas in the VGPR address space.
+def int_amdgcn_vgpr_lifetime_start :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+def int_amdgcn_vgpr_lifetime_end :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 84791f744cf25..4393269ffabc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
extern char &AMDGPULowerIdxOpsID;
+void initializeAMDGPUPrivateObjectVGPRsLegacyPass(PassRegistry &);
+extern char &AMDGPUPrivateObjectVGPRsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -266,7 +269,7 @@ void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
extern char &AMDGPUPreloadKernelArgumentsLegacyID;
// Passes common to R600 and SI
-FunctionPass *createAMDGPUPromoteAlloca();
+FunctionPass *createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel);
void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
extern char &AMDGPUPromoteAllocaID;
@@ -279,6 +282,17 @@ struct AMDGPUPromoteAllocaPass
TargetMachine &TM;
};
+/// Only allocates objects in the VGPR ("as memory") address space, which is
+/// required for functionality; used in place of AMDGPUPromoteAllocaPass when
+/// optimizations are disabled.
+struct AMDGPUVGPRAllocatePass : PassInfoMixin<AMDGPUVGPRAllocatePass> {
+ AMDGPUVGPRAllocatePass(TargetMachine &TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+ TargetMachine &TM;
+};
+
struct AMDGPUPromoteAllocaToVectorPass
: OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
@@ -447,6 +461,13 @@ class AMDGPULowerIdxOpsPass
MachineFunctionAnalysisManager &MFAM);
};
+class AMDGPUPrivateObjectVGPRsPass
+ : public RequiredPassInfoMixin<AMDGPUPrivateObjectVGPRsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0f88afbf20e37..5ba8c02f00b20 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -986,7 +986,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16);
- getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
+ // An alloca in the VGPR ("as memory") address space is custom-lowered to the
+ // address it was allocated; see legalizeFrameIndexVGPR.
+ getActionDefinitionsBuilder(G_FRAME_INDEX)
+ .legalFor({PrivatePtr})
+ .customFor({VGPRPtr});
// If the amount is divergent, we have to do a wave reduction to get the
// maximum value, so this is expanded during RegBankSelect.
@@ -2362,6 +2366,8 @@ bool AMDGPULegalizerInfo::legalizeCustom(
switch (MI.getOpcode()) {
case TargetOpcode::G_ADDRSPACE_CAST:
return legalizeAddrSpaceCast(MI, MRI, B);
+ case TargetOpcode::G_FRAME_INDEX:
+ return legalizeFrameIndexVGPR(MI, B);
case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
return legalizeFroundeven(MI, MRI, B);
case TargetOpcode::G_FCEIL:
@@ -3521,6 +3527,22 @@ static LLT widenToNextPowerOf2(LLT Ty) {
/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
/// Parallels the SelectionDAG LowerLoadStoreVGPR.
+// The address of an object in the VGPR ("as memory") address space is where
+// AMDGPUPromoteAlloca placed it, recorded on the alloca. Parallels the
+// SelectionDAG SITargetLowering::lowerFrameIndex.
+bool AMDGPULegalizerInfo::legalizeFrameIndexVGPR(MachineInstr &MI,
+ MachineIRBuilder &B) const {
+ MachineFunction &MF = B.getMF();
+ int FI = MI.getOperand(1).getIndex();
+ const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
+ assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR);
+
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ B.buildConstant(MI.getOperand(0), MD.getAddress());
+ MI.eraseFromParent();
+ return true;
+}
+
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
@@ -8942,6 +8964,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
return true;
}
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
+ unsigned Opc = IntrID == Intrinsic::amdgcn_vgpr_lifetime_start
+ ? AMDGPU::VGPR_LIFETIME_START
+ : AMDGPU::VGPR_LIFETIME_END;
+ B.buildInstr(Opc).cloneMemRefs(MI);
+ MI.eraseFromParent();
+ return true;
+ }
case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 30fd16930e6ae..1cc9119575731 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -41,6 +41,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
+ bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const;
+
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index b109d0f5b2389..c8af1830378dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -16,6 +16,8 @@
#include "AMDGPU.h"
#include "AMDGPUAsmPrinter.h"
#include "AMDGPUMachineFunctionInfo.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
#include "MCTargetDesc/AMDGPUInstPrinter.h"
#include "MCTargetDesc/AMDGPUMCExpr.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -435,8 +437,19 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
}
if (MI->isMetaInstruction()) {
- if (isVerbose())
+ if (isVerbose()) {
+ if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(MI)) {
+ const auto &MD =
+ AMDGPU::AllocatedVGPRsMetadata::get(Marker->getObject());
+ unsigned Begin = MD.getAddress() / 4;
+ unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4;
+ OutStreamer->emitRawComment(
+ Twine(" VGPR lifetime ") + (Marker->isStart() ? "start" : "end") +
+ ": v[" + Twine(Begin) + ":" + Twine(End) + "]");
+ return;
+ }
OutStreamer->emitRawComment(" meta instruction");
+ }
return;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index e99594062871a..5497eb74de3de 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -74,6 +74,25 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
}
};
+// Wrapper for the markers bounding the lifetime of an object in the VGPR "as
+// memory" (address space 13) address space. The object is named by the memory
+// operand, which is the only thing these carry until
+// AMDGPUPrivateObjectVGPRs gives them the registers it occupies.
+class VGPRLifetimeInst : public MachineInstr {
+public:
+ bool isStart() const { return getOpcode() == AMDGPU::VGPR_LIFETIME_START; }
+
+ const AllocaInst &getObject() const {
+ return *cast<AllocaInst>((*memoperands_begin())->getValue());
+ }
+
+ static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ return Opc == AMDGPU::VGPR_LIFETIME_START ||
+ Opc == AMDGPU::VGPR_LIFETIME_END;
+ }
+};
+
} // end namespace AMDGPUMI
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 7186ef6a5a827..7614fc23fac40 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -31,6 +31,32 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
GV->getValueType());
}
+AllocatedVGPRsMetadata &AllocatedVGPRsMetadata::get(const AllocaInst &Alloca) {
+ return *cast<AllocatedVGPRsMetadata>(
+ Alloca.getMetadata("amdgpu.allocated.vgprs"));
+}
+
+unsigned AllocatedVGPRsMetadata::getAddress() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(0))->getValue())
+ ->getZExtValue();
+}
+
+unsigned AllocatedVGPRsMetadata::getSize() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(1))->getValue())
+ ->getZExtValue();
+}
+
+bool AllocatedVGPRsMetadata::classof(const MDNode *N) {
+ if (N->getNumOperands() != 2)
+ return false;
+ for (int I = 0; I != 2; ++I) {
+ auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(I));
+ if (!C || !isa<ConstantInt>(C->getValue()))
+ return false;
+ }
+ return true;
+}
+
void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) {
SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
Source.getAllMetadata(MD);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index f81b677051561..db04c254feaeb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -12,11 +12,13 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/IR/Metadata.h"
namespace llvm {
struct Align;
class AAResults;
+class AllocaInst;
class DataLayout;
class GlobalVariable;
class LoadInst;
@@ -106,6 +108,23 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA);
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA);
+/// Convenience wrapper for !amdgpu.allocated.vgprs metadata, which records
+/// where in the VGPR ("as memory") address space an alloca was placed.
+///
+/// NOTE: Using `isa` with this class is not very meaningful.
+class AllocatedVGPRsMetadata : public MDNode {
+public:
+ static AllocatedVGPRsMetadata &get(const AllocaInst &Alloca);
+
+ /// Get the indicated (byte) address for the VGPR allocation.
+ unsigned getAddress() const;
+
+ /// Get the size (in bytes) of the VGPR allocation.
+ unsigned getSize() const;
+
+ static bool classof(const MDNode *N);
+};
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 534d97df145c0..e5385d189b94b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -73,6 +73,7 @@ FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
AMDGPUUnifyDivergentExitNodesPass())
+FUNCTION_PASS("amdgpu-vgpr-allocate", AMDGPUVGPRAllocatePass(*this))
FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this)))
FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
@@ -124,6 +125,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMA
MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
+MACHINE_FUNCTION_PASS("amdgpu-private-object-vgprs", AMDGPUPrivateObjectVGPRsPass())
MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
new file mode 100644
index 0000000000000..750988d23c2cd
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
@@ -0,0 +1,339 @@
+//===- AMDGPUPrivateObjectVGPRs.cpp - Private object VGPRs ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Mark the physical VGPRs an object in the VGPR ("as memory") address space
+/// was allocated to as used, so that register allocation does not hand them to
+/// anything else where the object is live:
+///
+/// * Add implicit use/def operands to the VGPR_LIFETIME_{START,END} pseudos
+/// * Add the VGPRs to basic block live-ins
+///
+/// The allocation itself is recorded on the alloca as !amdgpu.allocated.vgprs
+/// by AMDGPUPromoteAlloca, and reaches this pass through the memory operand of
+/// the lifetime markers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
+#include "GCNSubtarget.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-private-object-vgprs"
+
+using ObjectRegs = SmallVector<MCPhysReg, 50>;
+
+namespace {
+
+class AMDGPUPrivateObjectVGPRsImpl {
+public:
+ AMDGPUPrivateObjectVGPRsImpl(const MachineFunction &MF, LiveIntervals *LIS)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()), LIS(LIS) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ struct AllocaBBInfo {
+ const AllocaInst *Alloca = nullptr;
+ bool LiveIn = false;
+ bool Starts = false;
+ bool Ends = false;
+ };
+
+ ObjectRegs computeObjectRegs(const AllocaInst &Alloca) const;
+
+ const SIInstrInfo *TII;
+ LiveIntervals *LIS;
+
+ DenseMap<const AllocaInst *, std::pair<ObjectRegs, MachineMemOperand *>>
+ AllocaObjectRegs;
+};
+
+} // end anonymous namespace
+
+// The registers an object occupies: one per dword, starting at the register its
+// byte address in the address space names.
+ObjectRegs AMDGPUPrivateObjectVGPRsImpl::computeObjectRegs(
+ const AllocaInst &Alloca) const {
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(Alloca);
+ unsigned Offset = MD.getAddress();
+ assert(Offset % 4 == 0 && "Object is not dword aligned");
+
+ // An object that does not fill its last register still owns the whole of it.
+ unsigned BaseRegIdx = Offset / 4;
+ unsigned NumRegs = divideCeil(MD.getSize(), 4);
+
+ ObjectRegs Regs;
+ Regs.reserve(NumRegs);
+ for (unsigned I : seq(NumRegs))
+ Regs.push_back(AMDGPU::VGPR_32RegClass.getRegister(BaseRegIdx + I));
+
+ return Regs;
+}
+
+bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) {
+ // Sort basic blocks in reverse post-order for the live-out/live-in
+ // propagation.
+ DenseMap<MachineBasicBlock *, unsigned> BlockToIndex;
+ SmallVector<MachineBasicBlock *> IndexToBlock;
+ ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
+ for (MachineBasicBlock *MBB : RPOT) {
+ BlockToIndex[MBB] = IndexToBlock.size();
+ IndexToBlock.push_back(MBB);
+ }
+
+ // Fixed-point iteration to determine basic block live-ins.
+ //
+ // The first pass of the fixed-point iteration also scans instructions.
+ SmallVector<SmallVector<AllocaBBInfo>> BBInfos(IndexToBlock.size());
+ SmallBitVector Worklist(IndexToBlock.size());
+ bool Changed = false;
+
+ for (bool Dirty = true, FirstPass = true; Dirty; FirstPass = false) {
+ Dirty = false;
+
+ for (auto [MBBI, MBB] : enumerate(IndexToBlock)) {
+ auto &BBI = BBInfos[MBBI];
+
+ // During the first outer iteration, augment VGPR_LIFETIME_{START,END}
+ // with implicit operands and record the initial per-basic block
+ // information to compute live-ins.
+ if (FirstPass) {
+ for (MachineInstr &MI : *MBB) {
+ if (MI.getOpcode() != AMDGPU::VGPR_LIFETIME_START &&
+ MI.getOpcode() != AMDGPU::VGPR_LIFETIME_END)
+ continue;
+
+ bool IsStart = MI.getOpcode() == AMDGPU::VGPR_LIFETIME_START;
+ MachineMemOperand *MMO = *MI.memoperands_begin();
+ const auto *Alloca = cast<AllocaInst>(MMO->getValue());
+
+ auto ObjRegsIt = AllocaObjectRegs.find(Alloca);
+ if (ObjRegsIt == AllocaObjectRegs.end())
+ ObjRegsIt = AllocaObjectRegs
+ .try_emplace(Alloca, computeObjectRegs(*Alloca), MMO)
+ .first;
+
+ // The object comes into existence at the start and dies at the end.
+ for (MCPhysReg Reg : ObjRegsIt->second.first)
+ MI.addOperand(MachineOperand::CreateReg(
+ Reg, /*isDef=*/IsStart, /*isImp=*/true, /*isKill=*/!IsStart));
+
+ auto It = find_if(BBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == Alloca;
+ });
+ if (It == BBI.end()) {
+ BBI.push_back({Alloca, false, false, false});
+ It = std::prev(BBI.end());
+ }
+ It->Starts = IsStart;
+ It->Ends = !IsStart;
+
+ Changed = true;
+ }
+ } else {
+ if (!Worklist[MBBI])
+ continue;
+ Worklist[MBBI] = false;
+ }
+
+ // Propagate live-outs into successors.
+ for (const auto &ABBI : BBI) {
+ if (!((ABBI.LiveIn && !ABBI.Ends) || ABBI.Starts))
+ continue;
+
+ const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first;
+ for (MachineBasicBlock *Succ : MBB->successors()) {
+ unsigned SuccI = BlockToIndex.at(Succ);
+ auto &SuccBBI = BBInfos[SuccI];
+ auto It = find_if(SuccBBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == ABBI.Alloca;
+ });
+
+ bool Update = false;
+ if (It == SuccBBI.end()) {
+ SuccBBI.push_back({ABBI.Alloca, true, false, false});
+ It = std::prev(SuccBBI.end());
+ Update = true;
+ } else if (!It->LiveIn) {
+ It->LiveIn = true;
+ Update = true;
+ }
+
+ if (!Update)
+ continue;
+
+ // We are live-out from the successor because of the newly found
+ // live-in. If the successor is earlier in RPOT, we will have to
+ // re-evaluate it on the next outer iteration.
+ if (!It->Starts && !It->Ends && SuccI < MBBI) {
+ Worklist[SuccI] = true;
+ Dirty = true;
+ }
+
+ for (MCPhysReg Reg : Regs)
+ Succ->addLiveIn(Reg);
+ }
+ }
+ }
+ }
+
+ // It is legal for the pre-isel LLVM IR to have a lifetime.start without a
+ // lifetime.end. Liveness analysis is strong enough to mark physical registers
+ // as unused immediately after VGPR_LIFETIME_START in this case.
+ //
+ // Add VGPR_LIFETIME_END instructions at the end of basic blocks that end the
+ // function.
+ for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) {
+ if (!MBB->succ_empty())
+ continue;
+
+ for (const auto &ABBI : BBInfos[BBIdx]) {
+ if (ABBI.Ends || (!ABBI.LiveIn && !ABBI.Starts))
+ continue;
+
+ // There may be a COPY to a conflicting physical VGPR before a function
+ // return, so put the end as late as possible: walk back over the
+ // instructions that cannot be observing the object. Anything that touches
+ // memory, calls, or has side effects ends the walk, since the object has
+ // to stay reserved across it.
+ MachineBasicBlock::iterator IP = MBB->getFirstTerminator();
+ while (IP != MBB->begin()) {
+ --IP;
+ if (IP->mayStore() || IP->mayLoad() || IP->isCall() ||
+ IP->hasUnmodeledSideEffects()) {
+ ++IP;
+ break;
+ }
+ }
+
+ const auto &[ObjRegs, MMO] = AllocaObjectRegs.at(ABBI.Alloca);
+ MachineInstr *MI =
+ BuildMI(*MBB, IP, {}, TII->get(AMDGPU::VGPR_LIFETIME_END))
+ .addMemOperand(MMO);
+ for (MCPhysReg Reg : ObjRegs)
+ MI->addOperand(MachineOperand::CreateReg(
+ Reg, /*isDef=*/false, /*isImp=*/true, /*isKill=*/true));
+
+ Changed = true;
+ }
+ }
+
+ // An object lives in caller-saved registers, so a callee is free to overwrite
+ // it. Being live across a call is diagnosed rather than left to read back
+ // whatever the callee happened to leave behind. One diagnostic per object:
+ // the rest of its calls would say the same thing.
+ SmallPtrSet<const AllocaInst *, 4> Diagnosed;
+ for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) {
+ for (const auto &ABBI : BBInfos[BBIdx]) {
+ if (Diagnosed.contains(ABBI.Alloca))
+ continue;
+
+ bool Live = ABBI.LiveIn;
+ for (const MachineInstr &MI : *MBB) {
+ if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(&MI)) {
+ if (&Marker->getObject() == ABBI.Alloca)
+ Live = Marker->isStart();
+ continue;
+ }
+ if (!Live || !MI.isCall())
+ continue;
+
+ const Function &F = MF.getFunction();
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F,
+ "object in the VGPR 'as memory' address space (13) is live across "
+ "a call",
+ MI.getDebugLoc()));
+ Diagnosed.insert(ABBI.Alloca);
+ break;
+ }
+ }
+ }
+
+ // Remove live ranges from LiveIntervals. They will be recalculated lazily.
+ if (LIS) {
+ for (const auto &[Alloca, RegsAndMMO] : AllocaObjectRegs) {
+ for (MCPhysReg Reg : RegsAndMMO.first)
+ LIS->removeAllRegUnitsForPhysReg(Reg);
+ }
+ }
+
+ return Changed;
+}
+
+namespace {
+
+class AMDGPUPrivateObjectVGPRsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUPrivateObjectVGPRsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ LiveIntervals *LIS = nullptr;
+ if (auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>())
+ LIS = &LISWrapper->getLIS();
+ return AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ // LiveVariables only tracks virtual registers and we only touch physical
+ // registers.
+ AU.addPreserved<LiveVariablesWrapperPass>();
+ AU.addPreserved<SlotIndexesWrapperPass>();
+ AU.addPreserved<LiveIntervalsWrapperPass>();
+ AU.addPreservedID(MachineLoopInfoID);
+ AU.addPreservedID(MachineDominatorsID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override {
+ return "AMDGPU Def/use private object VGPRs";
+ }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPUPrivateObjectVGPRsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
+ if (!AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF))
+ return PreservedAnalyses::all();
+
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<LiveVariablesAnalysis>();
+ PA.preserve<SlotIndexesAnalysis>();
+ PA.preserve<LiveIntervalsAnalysis>();
+ return PA;
+}
+
+char AMDGPUPrivateObjectVGPRsLegacy::ID = 0;
+
+char &llvm::AMDGPUPrivateObjectVGPRsID = AMDGPUPrivateObjectVGPRsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPUPrivateObjectVGPRsLegacy, DEBUG_TYPE,
+ "AMDGPU Def/use private object VGPRs", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 2223b9d036fa1..841c93e9c2ef4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -26,6 +26,7 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "AMDGPUMemoryUtils.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/STLExtras.h"
@@ -142,6 +143,9 @@ class AMDGPUPromoteAllocaImpl {
bool IsAMDGCN = false;
bool IsAMDHSA = false;
+ /// Next free byte offset in the VGPR ("as memory") address space.
+ unsigned AllocVGPROffset = 0;
+
std::pair<Value *, Value *> getLocalSizeYZ(IRBuilder<> &Builder);
Value *getWorkitemID(IRBuilder<> &Builder, unsigned N);
@@ -167,6 +171,8 @@ class AMDGPUPromoteAllocaImpl {
void
finishDeferredAllocaToLDSPromotion(SetVector<IntrinsicInst *> &DeferredIntrs);
+ void allocateVgprs(AllocaAnalysis &AA);
+
void scoreAlloca(AllocaAnalysis &AA) const;
void setFunctionLimits(const Function &F);
@@ -179,7 +185,11 @@ class AMDGPUPromoteAllocaImpl {
IsAMDHSA = TT.getOS() == Triple::AMDHSA;
}
- bool run(Function &F, bool PromoteToLDS);
+ /// IsLatePass is true if invoked as a codegen pass and false if invoked from
+ /// the optimization pipeline ("amdgpu-promote-alloca-to-vector" pass). NoOpt
+ /// indicates to perform only the codegen work that is strictly required for
+ /// functionality, which is allocating objects in the VGPR address space.
+ bool run(Function &F, bool IsLatePass, bool NoOpt = false);
};
// FIXME: This can create globals so should be a module pass.
@@ -187,7 +197,8 @@ class AMDGPUPromoteAlloca : public FunctionPass {
public:
static char ID;
- AMDGPUPromoteAlloca() : FunctionPass(ID) {}
+ explicit AMDGPUPromoteAlloca(CodeGenOptLevel OptLevel)
+ : FunctionPass(ID), NoOpt(OptLevel == CodeGenOptLevel::None) {}
bool runOnFunction(Function &F) override {
if (skipFunction(F))
@@ -196,17 +207,22 @@ class AMDGPUPromoteAlloca : public FunctionPass {
return AMDGPUPromoteAllocaImpl(
TPC->getTM<TargetMachine>(), *F.getParent(),
getAnalysis<LoopInfoWrapperPass>().getLoopInfo())
- .run(F, /*PromoteToLDS*/ true);
+ .run(F, /*IsLatePass=*/true, NoOpt);
return false;
}
- StringRef getPassName() const override { return "AMDGPU Promote Alloca"; }
+ StringRef getPassName() const override {
+ return NoOpt ? "AMDGPU VGPR Allocate" : "AMDGPU Promote Alloca";
+ }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
AU.addRequired<LoopInfoWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
+
+private:
+ bool NoOpt;
};
static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
@@ -251,7 +267,7 @@ PreservedAnalyses AMDGPUPromoteAllocaPass::run(Function &F,
FunctionAnalysisManager &AM) {
auto &LI = AM.getResult<LoopAnalysis>(F);
bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
- .run(F, /*PromoteToLDS=*/true);
+ .run(F, /*IsLatePass=*/true);
if (Changed) {
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
@@ -264,7 +280,7 @@ PreservedAnalyses
AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) {
auto &LI = AM.getResult<LoopAnalysis>(F);
bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
- .run(F, /*PromoteToLDS=*/false);
+ .run(F, /*IsLatePass=*/false);
if (Changed) {
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
@@ -273,8 +289,21 @@ AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) {
return PreservedAnalyses::all();
}
-FunctionPass *llvm::createAMDGPUPromoteAlloca() {
- return new AMDGPUPromoteAlloca();
+FunctionPass *llvm::createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel) {
+ return new AMDGPUPromoteAlloca(OptLevel);
+}
+
+PreservedAnalyses AMDGPUVGPRAllocatePass::run(Function &F,
+ FunctionAnalysisManager &AM) {
+ auto &LI = AM.getResult<LoopAnalysis>(F);
+ bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
+ .run(F, /*IsLatePass=*/true, /*NoOpt=*/true);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
}
bool AMDGPUPromoteAllocaImpl::collectAllocaUses(AllocaAnalysis &AA) const {
@@ -367,10 +396,16 @@ void AMDGPUPromoteAllocaImpl::setFunctionLimits(const Function &F) {
VGPRBudgetRatio = PromoteAllocaToVectorVGPRRatio;
}
-bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
- if (DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector)
+bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
+ assert((!NoOpt || IsLatePass) && "NoOpt only makes sense for the late pass");
+
+ // Without optimizations only the allocation below is done, so the promotion
+ // options do not get a say.
+ if (!NoOpt && DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector)
return false;
+ const bool PromoteToLDS = IsLatePass && !NoOpt;
+
bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
MaxVGPRs = IsAMDGCN ? getMaxVGPRs(CurrentLocalMemUsage, TM, F) : 128;
setFunctionLimits(F);
@@ -391,6 +426,21 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
LLVM_DEBUG(dbgs() << "Analyzing: " << *AI << '\n');
AllocaAnalysis AA{AI};
+
+ // An alloca that is already in the VGPR ("as memory") address space is
+ // not promoted, only given a place in that address space, which only the
+ // codegen pass does. It happens whether or not optimizations are enabled,
+ // since the address space cannot be used without it.
+ if (AI->getAddressSpace() == AMDGPUAS::VGPR) {
+ if (IsLatePass)
+ Allocas.push_back(std::move(AA));
+ continue;
+ }
+
+ // Everything below this point is an optimization.
+ if (NoOpt)
+ continue;
+
if (collectAllocaUses(AA)) {
analyzePromoteToVector(AA);
if (PromoteToLDS)
@@ -403,8 +453,15 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
}
}
- stable_sort(Allocas,
- [](const auto &A, const auto &B) { return A.Score > B.Score; });
+ stable_sort(Allocas, [](const auto &A, const auto &B) {
+ // Prioritize allocas that are already in the VGPR address space, since
+ // their allocation must not fail.
+ bool AIsVGPR = A.Alloca->getAddressSpace() == AMDGPUAS::VGPR;
+ bool BIsVGPR = B.Alloca->getAddressSpace() == AMDGPUAS::VGPR;
+ if (AIsVGPR != BIsVGPR)
+ return AIsVGPR;
+ return A.Score > B.Score;
+ });
// clang-format off
LLVM_DEBUG(
@@ -417,6 +474,17 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
bool Changed = false;
SetVector<IntrinsicInst *> DeferredIntrs;
for (AllocaAnalysis &AA : Allocas) {
+ if (AA.Alloca->getAddressSpace() == AMDGPUAS::VGPR) {
+ allocateVgprs(AA);
+ std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL);
+ assert(Size); // Expected to succeed on non-array alloca.
+ const unsigned AllocaCost = Size->getFixedValue() * 8;
+ // These registers are no longer available for vectorization.
+ VectorizationBudget -= std::min(VectorizationBudget, AllocaCost);
+ Changed = true;
+ continue;
+ }
+
if (AA.Vector.Ty) {
std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL);
assert(Size); // Expected to succeed on non-array alloca.
@@ -451,6 +519,66 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
return Changed;
}
+// Give an alloca in the VGPR ("as memory") address space a place in that
+// address space, recorded on it as !amdgpu.allocated.vgprs, and lifetime
+// markers the backend understands. AMDGPUPrivateObjectVGPRs turns the two into
+// defs and uses of the physical registers the object occupies, which is what
+// keeps register allocation off them while it is live.
+void AMDGPUPromoteAllocaImpl::allocateVgprs(AllocaAnalysis &AA) {
+ LLVMContext &Ctx = Mod.getContext();
+ const unsigned AllocaSize =
+ DL.getTypeAllocSize(AA.Alloca->getAllocatedType()).getFixedValue();
+
+ // The generic lifetime intrinsics do not survive into the backend, so use
+ // the address-space specific ones instead. An object with no lifetime start
+ // is live from its definition.
+ bool HaveLifetimeStart = false;
+ for (Use &U : AA.Alloca->uses()) {
+ auto *II = dyn_cast<IntrinsicInst>(U.getUser());
+ if (!II)
+ continue;
+ if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ HaveLifetimeStart = true;
+ II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+ &Mod, Intrinsic::amdgcn_vgpr_lifetime_start, AA.Alloca->getType()));
+ } else if (II->getIntrinsicID() == Intrinsic::lifetime_end) {
+ II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+ &Mod, Intrinsic::amdgcn_vgpr_lifetime_end, AA.Alloca->getType()));
+ }
+ }
+
+ if (!HaveLifetimeStart) {
+ Instruction *IP = AA.Alloca->getNextNode();
+ while (isa<AllocaInst>(IP))
+ IP = IP->getNextNode();
+ IRBuilder<> B(IP);
+ B.SetCurrentDebugLocation(AA.Alloca->getDebugLoc());
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_vgpr_lifetime_start,
+ AA.Alloca);
+ }
+
+ // Allocating twice would place a second object on top of this one, so an
+ // object that already has its place keeps it, and the next one is allocated
+ // after it. The markers above are still rewritten, since IR that carries the
+ // metadata need not carry them.
+ if (MDNode *MD = AA.Alloca->getMetadata("amdgpu.allocated.vgprs")) {
+ const auto &Allocated = *cast<AMDGPU::AllocatedVGPRsMetadata>(MD);
+ AllocVGPROffset = std::max(
+ AllocVGPROffset, Allocated.getAddress() + alignTo(AllocaSize, 4));
+ return;
+ }
+
+ // Find space for the alloca. Objects are packed in order at dword
+ // granularity, since a register cannot be shared between two of them.
+ Type *I32 = Type::getInt32Ty(Ctx);
+ AA.Alloca->setMetadata(
+ "amdgpu.allocated.vgprs",
+ MDNode::get(
+ Ctx, {ConstantAsMetadata::get(ConstantInt::get(I32, AllocVGPROffset)),
+ ConstantAsMetadata::get(ConstantInt::get(I32, AllocaSize))}));
+ AllocVGPROffset += alignTo(AllocaSize, 4);
+}
+
// Checks if the instruction I is a memset user of the alloca AI that we can
// deal with. Currently, only non-volatile memsets that affect the whole alloca
// are handled.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f296570ecce8c..7831054335b3a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -669,6 +669,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
initializeAMDGPULowerIdxOpsLegacyPass(*PR);
+ initializeAMDGPUPrivateObjectVGPRsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1557,9 +1558,12 @@ void AMDGPUPassConfig::addIRPasses() {
addPass(createAtomicExpandLegacyPass());
- if (TM.getOptLevel() > CodeGenOptLevel::None) {
- addPass(createAMDGPUPromoteAlloca());
+ // With optimizations enabled, do the full promotion of allocas. Without
+ // optimizations, only allocate existing addrspace(13) allocas, which is
+ // needed for functionality rather than for performance.
+ addPass(createAMDGPUPromoteAlloca(TM.getOptLevel()));
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses();
@@ -1774,6 +1778,8 @@ void GCNPassConfig::addFastRegAlloc() {
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID);
+
insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID);
TargetPassConfig::addFastRegAlloc();
@@ -1806,6 +1812,8 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID);
+
if (EnableRewritePartialRegUses)
insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID);
@@ -2346,8 +2354,13 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
addFunctionPass(AtomicExpandPass(TM), PMW);
- if (TM.getOptLevel() > CodeGenOptLevel::None) {
+ // See the comment on the legacy pipeline above.
+ if (TM.getOptLevel() > CodeGenOptLevel::None)
addFunctionPass(AMDGPUPromoteAllocaPass(TM), PMW);
+ else
+ addFunctionPass(AMDGPUVGPRAllocatePass(TM), PMW);
+
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses(PMW);
@@ -2518,6 +2531,8 @@ void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const {
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass());
+
insertPass<TwoAddressInstructionPass>(SIWholeQuadModePass());
return Base::addFastRegAlloc(PMW);
@@ -2580,6 +2595,8 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass());
+
if (EnableRewritePartialRegUses)
insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass());
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 5c9f8a2063847..6853b3f47081c 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -92,6 +92,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUIGroupLP.cpp
AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
+ AMDGPUPrivateObjectVGPRs.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
AMDGPUMIRFormatter.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9f334be564f7f..33036d1d0cb56 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -278,6 +278,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
+
+ // Only to give an alloca in the VGPR ("as memory") address space the address
+ // it was allocated; every other frame index is left alone.
+ setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
setOperationAction(ISD::BlockAddress, {MVT::i32, MVT::i64}, Custom);
setOperationAction(ISD::ExternalSymbol, {MVT::i32, MVT::i64}, Custom);
@@ -1574,6 +1578,22 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Infos.push_back(Info);
return;
}
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ // A marker covers the whole object, whose size and placement within the
+ // VGPR ("as memory") address space AMDGPUPromoteAlloca recorded on the
+ // alloca. Carrying that as a memory operand is what lets
+ // AMDGPUPrivateObjectVGPRs find the registers the object occupies.
+ const auto *Alloca = cast<AllocaInst>(CI.getArgOperand(0));
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ Info.opc = ISD::INTRINSIC_VOID;
+ Info.memVT = EVT::getIntegerVT(CI.getContext(), MD.getSize() * 8);
+ Info.ptrVal = Alloca;
+ Info.flags = Flags | MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOStore;
+ Infos.push_back(Info);
+ return;
+ }
case Intrinsic::amdgcn_ds_append:
case Intrinsic::amdgcn_ds_consume: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
@@ -7575,6 +7595,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerBRCOND(Op, DAG);
case ISD::RETURNADDR:
return LowerRETURNADDR(Op, DAG);
+ case ISD::FrameIndex:
+ return lowerFrameIndex(Op, DAG);
case ISD::SPONENTRY:
return LowerSPONENTRY(Op, DAG);
case ISD::LOAD: {
@@ -10086,6 +10108,20 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
MachineMemOperand::MOInvariant);
}
+// The address of an object in the VGPR ("as memory") address space is where
+// AMDGPUPromoteAlloca placed it, recorded on the alloca. Every other frame
+// index keeps its default lowering.
+SDValue SITargetLowering::lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
+ MachineFunction &MF = DAG.getMachineFunction();
+ int FI = cast<FrameIndexSDNode>(Op)->getIndex();
+ const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
+ if (!Alloca || Alloca->getAddressSpace() != AMDGPUAS::VGPR)
+ return SDValue();
+
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ return DAG.getConstant(MD.getAddress(), SDLoc(Op), MVT::i32);
+}
+
SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
SelectionDAG &DAG) const {
// TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
@@ -12295,6 +12331,16 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
unsigned IntrinsicID = Op.getConstantOperandVal(1);
switch (IntrinsicID) {
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ unsigned Opcode = IntrinsicID == Intrinsic::amdgcn_vgpr_lifetime_start
+ ? AMDGPU::VGPR_LIFETIME_START
+ : AMDGPU::VGPR_LIFETIME_END;
+ MachineSDNode *Marker =
+ DAG.getMachineNode(Opcode, DL, MVT::Other, Chain);
+ DAG.setNodeMemRefs(Marker, {cast<MemSDNode>(Op)->getMemOperand()});
+ return SDValue(Marker, 0);
+ }
case Intrinsic::amdgcn_exp_compr: {
if (!Subtarget->hasCompressedExport()) {
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
@@ -19990,6 +20036,17 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
Info->limitOccupancy(MF);
+ // An object in the VGPR "as memory" address space lives in registers, and
+ // every reference to its frame index has been replaced by the address it was
+ // allocated, so it must not take up stack space as well.
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ for (int FI = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); FI != E;
+ ++FI) {
+ const AllocaInst *Alloca = MFI.getObjectAllocation(FI);
+ if (Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR)
+ MFI.RemoveStackObject(FI);
+ }
+
// Give a VGPR "as memory" indexed access its M0 operand, matching how
// AMDGPULowerVGPREncoding will expand it. Under the VGPR indexing mode the
// expansion emits an s_set_gpr_idx_on, which reads the index out of its SGPR
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f270a03422757..1f0008b392b3a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -182,6 +182,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
SelectionDAG &DAG) const;
+ SDValue lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d5d1e1f44fb99..c5ddc84ba13d7 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4064,13 +4064,16 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
return false;
// VGPR "as memory" indexed accesses only alias each other, and then only
- // when their [idx+offset, idx+offset+width) dword ranges overlap.
+ // when their [idx+offset, idx+offset+width) dword ranges overlap. The
+ // exception is a lifetime marker, which conceptually overwrites the whole
+ // object it covers and so may alias any access to it.
const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa);
const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb);
if (IsLdStIdxA || IsLdStIdxB) {
if (IsLdStIdxA && IsLdStIdxB)
return checkInstOffsetsDoNotOverlap(MIa, MIb);
- return true;
+ return !isa<AMDGPUMI::VGPRLifetimeInst>(MIa) &&
+ !isa<AMDGPUMI::VGPRLifetimeInst>(MIb);
}
if (isLDSDMA(MIa) || isLDSDMA(MIb))
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 1b66711c4fbe3..c836af2f36dd3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1161,6 +1161,25 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+/// Mark the start and end of the lifetime of an object in the VGPR ("as
+/// memory") address space. AMDGPUPrivateObjectVGPRs gives these the physical
+/// VGPRs the object was allocated to, as an implicit def on the start and an
+/// implicit killed use on the end, which is what keeps register allocation off
+/// those registers while the object is live.
+///
+/// These instructions conceptually overwrite the referenced memory with
+/// "undef".
+def VGPR_LIFETIME_START : SPseudoInstSI<(outs), (ins)> {
+ let hasSideEffects = false;
+ let mayStore = true;
+ let isMeta = true;
+}
+def VGPR_LIFETIME_END : SPseudoInstSI<(outs), (ins)> {
+ let hasSideEffects = false;
+ let mayStore = true;
+ let isMeta = true;
+}
+
// Used for byte or short loads.
// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
// sign-extended loads.
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
new file mode 100644
index 0000000000000..d18ed7488ab69
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
@@ -0,0 +1,70 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+
+; An object in the VGPR "as memory" address space (13) is allocated to
+; caller-saved registers, so a callee is free to overwrite it. Being live across
+; a call is therefore diagnosed rather than quietly reading back whatever the
+; callee left behind.
+
+declare void @extern_func()
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+
+; CHECK: error: {{.*}}in function across_call{{.*}}object in the VGPR 'as memory' address space (13) is live across a call
+define void @across_call(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void @extern_func()
+ %v = load volatile i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The lifetime ends before the call, so the registers are free by then.
+; CHECK-NOT: in function dies_before_call
+define void @dies_before_call(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load volatile i32, ptr addrspace(13) %p
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ call void @extern_func()
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Reachability, not textual order: the object is still live on the path through
+; the call.
+; CHECK: error: {{.*}}in function across_call_in_branch{{.*}}live across a call
+define void @across_call_in_branch(ptr addrspace(1) %out, i32 %i, i1 %c) {
+entry:
+ %obj = alloca [4 x i32], addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ br i1 %c, label %call, label %join
+
+call:
+ call void @extern_func()
+ br label %join
+
+join:
+ %v = load volatile i32, ptr addrspace(13) %p
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; An object with no lifetime end is live to the end of the function, so a call
+; in the exit block is across it too: the synthesized end goes after the call,
+; not before it.
+; CHECK: error: {{.*}}in function call_in_exit_block{{.*}}live across a call
+define void @call_in_exit_block(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void @extern_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
new file mode 100644
index 0000000000000..652798f0278a8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-promote-alloca -o - %s | FileCheck %s
+
+; An alloca in the VGPR "as memory" address space (13) is not promoted but
+; allocated: AMDGPUPromoteAlloca gives it a place in that address space,
+; recorded as !amdgpu.allocated.vgprs {byte offset, byte size}, and lifetime
+; markers the backend turns into defs and uses of the registers it occupies.
+
+define amdgpu_kernel void @one_object(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @one_object(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [4 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr addrspace(13) [[OBJ]], i32 0, i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Objects are packed in order, at dword granularity, since a register cannot be
+; shared between two of them.
+define amdgpu_kernel void @packed_objects(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @packed_objects(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca i8, align 1, addrspace(13), !amdgpu.allocated.vgprs [[META1:![0-9]+]]
+; CHECK-NEXT: [[B:%.*]] = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META2:![0-9]+]]
+; CHECK-NEXT: [[C:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META3:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[C]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[B]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[A]])
+; CHECK-NEXT: [[VA:%.*]] = load i8, ptr addrspace(13) [[A]], align 1
+; CHECK-NEXT: [[VB:%.*]] = load i32, ptr addrspace(13) [[B]], align 4
+; CHECK-NEXT: [[VC:%.*]] = load i32, ptr addrspace(13) [[C]], align 4
+; CHECK-NEXT: [[EA:%.*]] = zext i8 [[VA]] to i32
+; CHECK-NEXT: [[S0:%.*]] = add i32 [[EA]], [[VB]]
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[S0]], [[VC]]
+; CHECK-NEXT: store i32 [[S1]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca i8, addrspace(13)
+ %b = alloca [2 x i32], addrspace(13)
+ %c = alloca i32, addrspace(13)
+ %va = load i8, ptr addrspace(13) %a
+ %vb = load i32, ptr addrspace(13) %b
+ %vc = load i32, ptr addrspace(13) %c
+ %ea = zext i8 %va to i32
+ %s0 = add i32 %ea, %vb
+ %s1 = add i32 %s0, %vc
+ store i32 %s1, ptr addrspace(1) %out
+ ret void
+}
+
+; Existing lifetime markers become the address space specific ones rather than
+; a synthesized start.
+define amdgpu_kernel void @existing_lifetimes(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @existing_lifetimes(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META4:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ ret void
+}
+
+; Running again must not allocate a second place on top of the first, and must
+; not add a second marker.
+define amdgpu_kernel void @already_allocated(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @already_allocated(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META5:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !0
+ call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Metadata without the address-space specific markers: the place is kept, but
+; the generic markers still have to be rewritten, or nothing reserves the
+; registers in the backend.
+define amdgpu_kernel void @allocated_but_generic_markers(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @allocated_but_generic_markers(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META6:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !1
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ ret void
+}
+
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+declare void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) nocapture)
+
+!0 = !{i32 8, i32 4}
+!1 = !{i32 32, i32 4}
+;.
+; CHECK: [[META0]] = !{i32 0, i32 16}
+; CHECK: [[META1]] = !{i32 0, i32 1}
+; CHECK: [[META2]] = !{i32 4, i32 8}
+; CHECK: [[META3]] = !{i32 12, i32 4}
+; CHECK: [[META4]] = !{i32 0, i32 4}
+; CHECK: [[META5]] = !{i32 8, i32 4}
+; CHECK: [[META6]] = !{i32 32, i32 4}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
new file mode 100644
index 0000000000000..f1b9b8d378cdb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Allocating the object is needed for functionality, not performance, so it
+; happens without optimizations too.
+; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT
+; RUN: llc -O0 -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT
+
+; NOOPT: avoids_object_registers:
+; NOOPT: ; VGPR lifetime start: v[0:1]
+; NOOPT: v_movrel{{[sd]}}_b32
+; NOOPT: ; VGPR lifetime end: v[0:1]
+; NOOPT: ScratchSize: 0
+
+; The registers an object in the VGPR "as memory" address space (13) was
+; allocated must not be handed to anything else while it is live. The lifetime
+; markers carry them as an implicit def and an implicit killed use, so the
+; register allocator sees the object the same way it sees any other value.
+
+; The object occupies v[0:1], so the values live across it are allocated
+; elsewhere.
+define amdgpu_kernel void @avoids_object_registers(ptr addrspace(1) %out, i32 %i) {
+; GFX12-SDAG-LABEL: avoids_object_registers:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:1]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v2
+; GFX12-SDAG-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:1]
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, 7, v2
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: avoids_object_registers:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:1]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v2
+; GFX12-GISEL-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v4
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s4, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:1]
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, s2, v2
+; GFX12-GISEL-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca [2 x i32], addrspace(13)
+ %p = getelementptr [2 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %a = load volatile i32, ptr addrspace(1) %out
+ %b = load volatile i32, ptr addrspace(1) %out
+ %c = load volatile i32, ptr addrspace(1) %out
+ %v = load i32, ptr addrspace(13) %p
+ %s0 = add i32 %a, %b
+ %s1 = add i32 %s0, %c
+ %s2 = add i32 %s1, %v
+ store i32 %s2, ptr addrspace(1) %out
+ ret void
+}
+
+; The object lives in registers, so it takes no stack space.
+define amdgpu_kernel void @no_scratch(ptr addrspace(1) %out, i32 %i) {
+; GFX12-SDAG-LABEL: no_scratch:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 7
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:3]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: global_store_b32 v4, v5, s[0:1]
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v5
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: no_scratch:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v4, 7 :: v_dual_mov_b32 v5, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:3]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s2, 0x1e0000
+; GFX12-GISEL-NEXT: global_store_b32 v5, v4, s[0:1]
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v4
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:3]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Registers are only unavailable where the object is live: after its end they
+; can be used again.
+define amdgpu_kernel void @registers_reused_after_end(ptr addrspace(1) %out) {
+; GFX12-SDAG-LABEL: registers_reused_after_end:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:0]
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:0]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: global_store_b32 v1, v2, s[0:1]
+; GFX12-SDAG-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: registers_reused_after_end:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_mov_b32 m0, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:0]
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:0]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: global_store_b32 v1, v2, s[0:1]
+; GFX12-GISEL-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca i32, addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ %a = load volatile i32, ptr addrspace(1) %out
+ %b = load volatile i32, ptr addrspace(1) %out
+ %s = add i32 %a, %b
+ store i32 %s, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index f465ee4c0470c..e66de5ade3a63 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -29,6 +29,7 @@
; GCN-O0-NEXT: amdgpu-lower-module-lds
; GCN-O0-NEXT: function
; GCN-O0-NEXT: atomic-expand
+; GCN-O0-NEXT: amdgpu-vgpr-allocate
; GCN-O0-NEXT: verify
; GCN-O0-NEXT: unreachableblockelim
; GCN-O0-NEXT: ee-instrument<post-inline>
@@ -72,6 +73,7 @@
; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
+; GCN-O0-NEXT: amdgpu-private-object-vgprs
; GCN-O0-NEXT: two-address-instruction
; GCN-O0-NEXT: si-wqm
; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg
@@ -230,6 +232,7 @@
; GCN-O2-NEXT: require<machine-loops>
; GCN-O2-NEXT: phi-node-elimination
; GCN-O2-NEXT: si-lower-control-flow
+; GCN-O2-NEXT: amdgpu-private-object-vgprs
; GCN-O2-NEXT: two-address-instruction
; GCN-O2-NEXT: register-coalescer
; GCN-O2-NEXT: rename-independent-subregs
@@ -419,6 +422,7 @@
; GCN-O3-NEXT: require<machine-loops>
; GCN-O3-NEXT: phi-node-elimination
; GCN-O3-NEXT: si-lower-control-flow
+; GCN-O3-NEXT: amdgpu-private-object-vgprs
; GCN-O3-NEXT: two-address-instruction
; GCN-O3-NEXT: register-coalescer
; GCN-O3-NEXT: rename-independent-subregs
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 1fd95e2289480..0cdd53c890d32 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -49,11 +49,13 @@
; GCN-O0-NEXT: Lower uses of LDS variables from non-kernel functions
; GCN-O0-NEXT: FunctionPass Manager
; GCN-O0-NEXT: Expand Atomic instructions
+; GCN-O0-NEXT: Dominator Tree Construction
+; GCN-O0-NEXT: Natural Loop Information
+; GCN-O0-NEXT: AMDGPU VGPR Allocate
; GCN-O0-NEXT: Remove unreachable blocks from the CFG
; GCN-O0-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O0-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O0-NEXT: Expand reduction intrinsics
-; GCN-O0-NEXT: Dominator Tree Construction
; GCN-O0-NEXT: AMDGPU Lower Kernel Arguments
; GCN-O0-NEXT: Lower buffer fat pointer operations to buffer resources
; GCN-O0-NEXT: FunctionPass Manager
@@ -116,6 +118,7 @@
; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
+; GCN-O0-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O0-NEXT: Two-Address instruction pass
; GCN-O0-NEXT: MachineDominator Tree Construction
; GCN-O0-NEXT: Slot index numbering
@@ -364,6 +367,7 @@
; GCN-O1-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O1-NEXT: Two-Address instruction pass
; GCN-O1-NEXT: Slot index numbering
; GCN-O1-NEXT: Live Interval Analysis
@@ -686,6 +690,7 @@
; GCN-O1-OPTS-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-OPTS-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-OPTS-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-OPTS-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O1-OPTS-NEXT: Two-Address instruction pass
; GCN-O1-OPTS-NEXT: Slot index numbering
; GCN-O1-OPTS-NEXT: Live Interval Analysis
@@ -1013,6 +1018,7 @@
; GCN-O2-NEXT: SI Optimize VGPR LiveRange
; GCN-O2-NEXT: Eliminate PHI nodes for register allocation
; GCN-O2-NEXT: SI Lower control flow pseudo instructions
+; GCN-O2-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O2-NEXT: Two-Address instruction pass
; GCN-O2-NEXT: Slot index numbering
; GCN-O2-NEXT: Live Interval Analysis
@@ -1354,6 +1360,7 @@
; GCN-O3-NEXT: SI Optimize VGPR LiveRange
; GCN-O3-NEXT: Eliminate PHI nodes for register allocation
; GCN-O3-NEXT: SI Lower control flow pseudo instructions
+; GCN-O3-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O3-NEXT: Two-Address instruction pass
; GCN-O3-NEXT: Slot index numbering
; GCN-O3-NEXT: Live Interval Analysis
More information about the llvm-branch-commits
mailing list