[llvm-branch-commits] [llvm] [AMDGPU] Allocate and reserve registers for address space 13 objects (PR #214248)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 5 08:13:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Gheorghe-Teodor Bercea (doru1004)
<details>
<summary>Changes</summary>
Allocate and reserve registers for address space 13 objects
---
Patch is 70.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214248.diff
22 Files Affected:
- (modified) llvm/include/llvm/IR/IntrinsicsAMDGPU.td (+9)
- (modified) llvm/lib/Target/AMDGPU/AMDGPU.h (+22-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+33-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h (+2)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp (+14-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h (+19)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp (+26)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h (+19)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def (+2)
- (added) llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp (+339)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp (+140-12)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp (+20-3)
- (modified) llvm/lib/Target/AMDGPU/CMakeLists.txt (+1)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+57)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (+1)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+5-2)
- (modified) llvm/lib/Target/AMDGPU/SIInstructions.td (+19)
- (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll (+70)
- (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll (+133)
- (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll (+178)
- (modified) llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/llc-pipeline.ll (+8-1)
``````````diff
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 21882e247c027..1048f99752643 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -4292,4 +4292,13 @@ def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic<
/// incoming block.
def int_amdgcn_dead: DefaultAttrsIntrinsic<[llvm_any_ty], [],
[IntrNoMem]>;
+
+/// Backend-only intrinsics analogous to llvm.lifetime.{start,end} for marking
+/// the lifetimes of allocas in the VGPR address space.
+def int_amdgcn_vgpr_lifetime_start :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+def int_amdgcn_vgpr_lifetime_end :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 84791f744cf25..4393269ffabc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
extern char &AMDGPULowerIdxOpsID;
+void initializeAMDGPUPrivateObjectVGPRsLegacyPass(PassRegistry &);
+extern char &AMDGPUPrivateObjectVGPRsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -266,7 +269,7 @@ void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
extern char &AMDGPUPreloadKernelArgumentsLegacyID;
// Passes common to R600 and SI
-FunctionPass *createAMDGPUPromoteAlloca();
+FunctionPass *createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel);
void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
extern char &AMDGPUPromoteAllocaID;
@@ -279,6 +282,17 @@ struct AMDGPUPromoteAllocaPass
TargetMachine &TM;
};
+/// Only allocates objects in the VGPR ("as memory") address space, which is
+/// required for functionality; used in place of AMDGPUPromoteAllocaPass when
+/// optimizations are disabled.
+struct AMDGPUVGPRAllocatePass : PassInfoMixin<AMDGPUVGPRAllocatePass> {
+ AMDGPUVGPRAllocatePass(TargetMachine &TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+ TargetMachine &TM;
+};
+
struct AMDGPUPromoteAllocaToVectorPass
: OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
@@ -447,6 +461,13 @@ class AMDGPULowerIdxOpsPass
MachineFunctionAnalysisManager &MFAM);
};
+class AMDGPUPrivateObjectVGPRsPass
+ : public RequiredPassInfoMixin<AMDGPUPrivateObjectVGPRsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0f88afbf20e37..5ba8c02f00b20 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -986,7 +986,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16);
- getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
+ // An alloca in the VGPR ("as memory") address space is custom-lowered to the
+ // address it was allocated; see legalizeFrameIndexVGPR.
+ getActionDefinitionsBuilder(G_FRAME_INDEX)
+ .legalFor({PrivatePtr})
+ .customFor({VGPRPtr});
// If the amount is divergent, we have to do a wave reduction to get the
// maximum value, so this is expanded during RegBankSelect.
@@ -2362,6 +2366,8 @@ bool AMDGPULegalizerInfo::legalizeCustom(
switch (MI.getOpcode()) {
case TargetOpcode::G_ADDRSPACE_CAST:
return legalizeAddrSpaceCast(MI, MRI, B);
+ case TargetOpcode::G_FRAME_INDEX:
+ return legalizeFrameIndexVGPR(MI, B);
case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
return legalizeFroundeven(MI, MRI, B);
case TargetOpcode::G_FCEIL:
@@ -3521,6 +3527,22 @@ static LLT widenToNextPowerOf2(LLT Ty) {
/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
/// Parallels the SelectionDAG LowerLoadStoreVGPR.
+// The address of an object in the VGPR ("as memory") address space is where
+// AMDGPUPromoteAlloca placed it, recorded on the alloca. Parallels the
+// SelectionDAG SITargetLowering::lowerFrameIndex.
+bool AMDGPULegalizerInfo::legalizeFrameIndexVGPR(MachineInstr &MI,
+ MachineIRBuilder &B) const {
+ MachineFunction &MF = B.getMF();
+ int FI = MI.getOperand(1).getIndex();
+ const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
+ assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR);
+
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ B.buildConstant(MI.getOperand(0), MD.getAddress());
+ MI.eraseFromParent();
+ return true;
+}
+
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
@@ -8942,6 +8964,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
return true;
}
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
+ unsigned Opc = IntrID == Intrinsic::amdgcn_vgpr_lifetime_start
+ ? AMDGPU::VGPR_LIFETIME_START
+ : AMDGPU::VGPR_LIFETIME_END;
+ B.buildInstr(Opc).cloneMemRefs(MI);
+ MI.eraseFromParent();
+ return true;
+ }
case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 30fd16930e6ae..1cc9119575731 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -41,6 +41,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
+ bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const;
+
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index b109d0f5b2389..c8af1830378dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -16,6 +16,8 @@
#include "AMDGPU.h"
#include "AMDGPUAsmPrinter.h"
#include "AMDGPUMachineFunctionInfo.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
#include "MCTargetDesc/AMDGPUInstPrinter.h"
#include "MCTargetDesc/AMDGPUMCExpr.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -435,8 +437,19 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
}
if (MI->isMetaInstruction()) {
- if (isVerbose())
+ if (isVerbose()) {
+ if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(MI)) {
+ const auto &MD =
+ AMDGPU::AllocatedVGPRsMetadata::get(Marker->getObject());
+ unsigned Begin = MD.getAddress() / 4;
+ unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4;
+ OutStreamer->emitRawComment(
+ Twine(" VGPR lifetime ") + (Marker->isStart() ? "start" : "end") +
+ ": v[" + Twine(Begin) + ":" + Twine(End) + "]");
+ return;
+ }
OutStreamer->emitRawComment(" meta instruction");
+ }
return;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index e99594062871a..5497eb74de3de 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -74,6 +74,25 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
}
};
+// Wrapper for the markers bounding the lifetime of an object in the VGPR "as
+// memory" (address space 13) address space. The object is named by the memory
+// operand, which is the only thing these carry until
+// AMDGPUPrivateObjectVGPRs gives them the registers it occupies.
+class VGPRLifetimeInst : public MachineInstr {
+public:
+ bool isStart() const { return getOpcode() == AMDGPU::VGPR_LIFETIME_START; }
+
+ const AllocaInst &getObject() const {
+ return *cast<AllocaInst>((*memoperands_begin())->getValue());
+ }
+
+ static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ return Opc == AMDGPU::VGPR_LIFETIME_START ||
+ Opc == AMDGPU::VGPR_LIFETIME_END;
+ }
+};
+
} // end namespace AMDGPUMI
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 7186ef6a5a827..7614fc23fac40 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -31,6 +31,32 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
GV->getValueType());
}
+AllocatedVGPRsMetadata &AllocatedVGPRsMetadata::get(const AllocaInst &Alloca) {
+ return *cast<AllocatedVGPRsMetadata>(
+ Alloca.getMetadata("amdgpu.allocated.vgprs"));
+}
+
+unsigned AllocatedVGPRsMetadata::getAddress() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(0))->getValue())
+ ->getZExtValue();
+}
+
+unsigned AllocatedVGPRsMetadata::getSize() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(1))->getValue())
+ ->getZExtValue();
+}
+
+bool AllocatedVGPRsMetadata::classof(const MDNode *N) {
+ if (N->getNumOperands() != 2)
+ return false;
+ for (int I = 0; I != 2; ++I) {
+ auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(I));
+ if (!C || !isa<ConstantInt>(C->getValue()))
+ return false;
+ }
+ return true;
+}
+
void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) {
SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
Source.getAllMetadata(MD);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index f81b677051561..db04c254feaeb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -12,11 +12,13 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/IR/Metadata.h"
namespace llvm {
struct Align;
class AAResults;
+class AllocaInst;
class DataLayout;
class GlobalVariable;
class LoadInst;
@@ -106,6 +108,23 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA);
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA);
+/// Convenience wrapper for !amdgpu.allocated.vgprs metadata, which records
+/// where in the VGPR ("as memory") address space an alloca was placed.
+///
+/// NOTE: Using `isa` with this class is not very meaningful.
+class AllocatedVGPRsMetadata : public MDNode {
+public:
+ static AllocatedVGPRsMetadata &get(const AllocaInst &Alloca);
+
+ /// Get the indicated (byte) address for the VGPR allocation.
+ unsigned getAddress() const;
+
+ /// Get the size (in bytes) of the VGPR allocation.
+ unsigned getSize() const;
+
+ static bool classof(const MDNode *N);
+};
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 534d97df145c0..e5385d189b94b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -73,6 +73,7 @@ FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
AMDGPUUnifyDivergentExitNodesPass())
+FUNCTION_PASS("amdgpu-vgpr-allocate", AMDGPUVGPRAllocatePass(*this))
FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this)))
FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
@@ -124,6 +125,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMA
MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
+MACHINE_FUNCTION_PASS("amdgpu-private-object-vgprs", AMDGPUPrivateObjectVGPRsPass())
MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
new file mode 100644
index 0000000000000..750988d23c2cd
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
@@ -0,0 +1,339 @@
+//===- AMDGPUPrivateObjectVGPRs.cpp - Private object VGPRs ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Mark the physical VGPRs an object in the VGPR ("as memory") address space
+/// was allocated to as used, so that register allocation does not hand them to
+/// anything else where the object is live:
+///
+/// * Add implicit use/def operands to the VGPR_LIFETIME_{START,END} pseudos
+/// * Add the VGPRs to basic block live-ins
+///
+/// The allocation itself is recorded on the alloca as !amdgpu.allocated.vgprs
+/// by AMDGPUPromoteAlloca, and reaches this pass through the memory operand of
+/// the lifetime markers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
+#include "GCNSubtarget.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-private-object-vgprs"
+
+using ObjectRegs = SmallVector<MCPhysReg, 50>;
+
+namespace {
+
+class AMDGPUPrivateObjectVGPRsImpl {
+public:
+ AMDGPUPrivateObjectVGPRsImpl(const MachineFunction &MF, LiveIntervals *LIS)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()), LIS(LIS) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ struct AllocaBBInfo {
+ const AllocaInst *Alloca = nullptr;
+ bool LiveIn = false;
+ bool Starts = false;
+ bool Ends = false;
+ };
+
+ ObjectRegs computeObjectRegs(const AllocaInst &Alloca) const;
+
+ const SIInstrInfo *TII;
+ LiveIntervals *LIS;
+
+ DenseMap<const AllocaInst *, std::pair<ObjectRegs, MachineMemOperand *>>
+ AllocaObjectRegs;
+};
+
+} // end anonymous namespace
+
+// The registers an object occupies: one per dword, starting at the register its
+// byte address in the address space names.
+ObjectRegs AMDGPUPrivateObjectVGPRsImpl::computeObjectRegs(
+ const AllocaInst &Alloca) const {
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(Alloca);
+ unsigned Offset = MD.getAddress();
+ assert(Offset % 4 == 0 && "Object is not dword aligned");
+
+ // An object that does not fill its last register still owns the whole of it.
+ unsigned BaseRegIdx = Offset / 4;
+ unsigned NumRegs = divideCeil(MD.getSize(), 4);
+
+ ObjectRegs Regs;
+ Regs.reserve(NumRegs);
+ for (unsigned I : seq(NumRegs))
+ Regs.push_back(AMDGPU::VGPR_32RegClass.getRegister(BaseRegIdx + I));
+
+ return Regs;
+}
+
+bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) {
+ // Sort basic blocks in reverse post-order for the live-out/live-in
+ // propagation.
+ DenseMap<MachineBasicBlock *, unsigned> BlockToIndex;
+ SmallVector<MachineBasicBlock *> IndexToBlock;
+ ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
+ for (MachineBasicBlock *MBB : RPOT) {
+ BlockToIndex[MBB] = IndexToBlock.size();
+ IndexToBlock.push_back(MBB);
+ }
+
+ // Fixed-point iteration to determine basic block live-ins.
+ //
+ // The first pass of the fixed-point iteration also scans instructions.
+ SmallVector<SmallVector<AllocaBBInfo>> BBInfos(IndexToBlock.size());
+ SmallBitVector Worklist(IndexToBlock.size());
+ bool Changed = false;
+
+ for (bool Dirty = true, FirstPass = true; Dirty; FirstPass = false) {
+ Dirty = false;
+
+ for (auto [MBBI, MBB] : enumerate(IndexToBlock)) {
+ auto &BBI = BBInfos[MBBI];
+
+ // During the first outer iteration, augment VGPR_LIFETIME_{START,END}
+ // with implicit operands and record the initial per-basic block
+ // information to compute live-ins.
+ if (FirstPass) {
+ for (MachineInstr &MI : *MBB) {
+ if (MI.getOpcode() != AMDGPU::VGPR_LIFETIME_START &&
+ MI.getOpcode() != AMDGPU::VGPR_LIFETIME_END)
+ continue;
+
+ bool IsStart = MI.getOpcode() == AMDGPU::VGPR_LIFETIME_START;
+ MachineMemOperand *MMO = *MI.memoperands_begin();
+ const auto *Alloca = cast<AllocaInst>(MMO->getValue());
+
+ auto ObjRegsIt = AllocaObjectRegs.find(Alloca);
+ if (ObjRegsIt == AllocaObjectRegs.end())
+ ObjRegsIt = AllocaObjectRegs
+ .try_emplace(Alloca, computeObjectRegs(*Alloca), MMO)
+ .first;
+
+ // The object comes into existence at the start and dies at the end.
+ for (MCPhysReg Reg : ObjRegsIt->second.first)
+ MI.addOperand(MachineOperand::CreateReg(
+ Reg, /*isDef=*/IsStart, /*isImp=*/true, /*isKill=*/!IsStart));
+
+ auto It = find_if(BBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == Alloca;
+ });
+ if (It == BBI.end()) {
+ BBI.push_back({Alloca, false, false, false});
+ It = std::prev(BBI.end());
+ }
+ It->Starts = IsStart;
+ It->Ends = !IsStart;
+
+ Changed = true;
+ }
+ } else {
+ if (!Worklist[MBBI])
+ continue;
+ Worklist[MBBI] = false;
+ }
+
+ // Propagate live-outs into successors.
+ for (const auto &ABBI : BBI) {
+ if (!((ABBI.LiveIn && !ABBI.Ends) || ABBI.Starts))
+ continue;
+
+ const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first;
+ for (MachineBasicBlock *Succ : MBB->successors()) {
+ unsigned SuccI = BlockToIndex.at(Succ);
+ auto &SuccBBI = BBInfos[SuccI];
+ auto It = find_if(SuccBBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == ABBI.Alloca;
+ });
+
+ bool Update = false;
+ if (It == SuccBBI.end()) {
+ SuccBBI.push_back({ABBI.Alloca, true, false, false});
+ It = std::prev(SuccBBI.end());
+ Update = true;
+ } else if (!It->LiveIn) {
+ It->LiveIn = true;
+ Update = true;
+ }
+
+ if (!Update)
+ continue;
+
+ // We are...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214248
More information about the llvm-branch-commits
mailing list