[llvm-branch-commits] [llvm] [AMDGPU] Allocate and reserve registers for address space 13 objects (PR #214248)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 14 08:13:46 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/214248
>From ce465c0df381adcf2182bdd4ec409302198451a9 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Wed, 5 Aug 2026 17:57:28 +0300
Subject: [PATCH 1/3] Allocate and reserve registers for address space 13
objects
---
llvm/docs/AMDGPUUsage.rst | 44 ++-
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 9 +
llvm/lib/Target/AMDGPU/AMDGPU.h | 23 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 34 +-
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 2 +
llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 15 +-
llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 20 ++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 26 ++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 19 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 2 +
.../AMDGPU/AMDGPUPrivateObjectVGPRs.cpp | 340 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 152 +++++++-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 23 +-
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 56 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +-
llvm/lib/Target/AMDGPU/SIInstructions.td | 19 +
.../AddressSpaceVGPR/as-vgpr-across-call.ll | 80 +++++
.../AddressSpaceVGPR/as-vgpr-allocate.ll | 133 +++++++
.../as-vgpr-object-regalloc.ll | 178 +++++++++
.../as-vgpr-private-object-vgprs.mir | 106 ++++++
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 4 +
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 9 +-
24 files changed, 1280 insertions(+), 23 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-private-object-vgprs.mir
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 321d97e47b1c7..c10fe6875bcf1 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2143,6 +2143,24 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
* Local pointer to the LDS barrier data.
+ llvm.amdgcn.vgpr.lifetime.start Marks the start of the lifetime of an object in the
+ :ref:`VGPR<amdgpu-address-spaces>` address space, whose
+ placement within it is recorded by
+ :ref:`amdgpu.allocated.vgprs<amdgpu_allocated_vgprs>`.
+ Takes the ``alloca`` as its only argument.
+
+ This is a backend-only intrinsic, created from
+ ``llvm.lifetime.start`` by the ``amdgpu-promote-alloca``
+ pass; frontends should not emit it. The registers the
+ object occupies are marked as used between it and the
+ matching end, so register allocation does not hand them
+ to anything else there. An object with no start is live
+ from its definition.
+
+ llvm.amdgcn.vgpr.lifetime.end Marks the end of such a lifetime; see
+ ``llvm.amdgcn.vgpr.lifetime.start``. An object with no
+ end is live to the end of the function.
+
============================================== ==========================================================
.. TODO::
@@ -2516,8 +2534,30 @@ whether to skip the DPP optimization.
; Few expected active lanes: the atomic optimizer may skip the DPP optimization.
%old0 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 4}
- ; Many expected active lanes: the DPP optimization is not skipped.
- %old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
+; Many expected active lanes: the DPP optimization is not skipped.
+%old1 = atomicrmw add ptr addrspace(3) @lds, i32 %val acq_rel, !amdgpu.expected.active.lanes !{i32 32}
+
+.. _amdgpu_allocated_vgprs:
+
+'``amdgpu.allocated.vgprs``' Metadata
+-------------------------------------
+
+Records where in the :ref:`VGPR <amdgpu-address-spaces>` address space an
+``alloca`` in that address space was placed. It has two ``i32`` constant
+operands: the byte address of the object within the address space, and its size
+in bytes. Both are multiples of four for the address, since an object cannot
+share a register with another one.
+
+The metadata is added by the ``amdgpu-promote-alloca`` pass and is not intended
+to be written by frontends. The backend uses it in two ways: the address of the
+``alloca`` lowers to the constant recorded here rather than to a stack slot, and
+the physical registers the object occupies are derived from it so that register
+allocation can be kept off them while the object is live.
+
+.. code-block:: llvm
+
+ ; An object of four dwords placed at v[0:3].
+ %obj = alloca [4 x i32], addrspace(13), !amdgpu.allocated.vgprs !{i32 0, i32 16}
LLVM IR Attributes
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 2d4999ae2005c..d98c984580692 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -4363,4 +4363,13 @@ def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic<
/// incoming block.
def int_amdgcn_dead: DefaultAttrsIntrinsic<[llvm_any_ty], [],
[IntrNoMem]>;
+
+/// Backend-only intrinsics analogous to llvm.lifetime.{start,end} for marking
+/// the lifetimes of allocas in the VGPR address space.
+def int_amdgcn_vgpr_lifetime_start :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+def int_amdgcn_vgpr_lifetime_end :
+ DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+ [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 84791f744cf25..4393269ffabc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID;
void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &);
extern char &AMDGPULowerIdxOpsID;
+void initializeAMDGPUPrivateObjectVGPRsLegacyPass(PassRegistry &);
+extern char &AMDGPUPrivateObjectVGPRsID;
+
void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
extern char &SILowerSGPRSpillsLegacyID;
@@ -266,7 +269,7 @@ void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
extern char &AMDGPUPreloadKernelArgumentsLegacyID;
// Passes common to R600 and SI
-FunctionPass *createAMDGPUPromoteAlloca();
+FunctionPass *createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel);
void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
extern char &AMDGPUPromoteAllocaID;
@@ -279,6 +282,17 @@ struct AMDGPUPromoteAllocaPass
TargetMachine &TM;
};
+/// Only allocates objects in the VGPR ("as memory") address space, which is
+/// required for functionality; used in place of AMDGPUPromoteAllocaPass when
+/// optimizations are disabled.
+struct AMDGPUVGPRAllocatePass : PassInfoMixin<AMDGPUVGPRAllocatePass> {
+ AMDGPUVGPRAllocatePass(TargetMachine &TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+ TargetMachine &TM;
+};
+
struct AMDGPUPromoteAllocaToVectorPass
: OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
@@ -447,6 +461,13 @@ class AMDGPULowerIdxOpsPass
MachineFunctionAnalysisManager &MFAM);
};
+class AMDGPUPrivateObjectVGPRsPass
+ : public RequiredPassInfoMixin<AMDGPUPrivateObjectVGPRsPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
class SIInsertWaitcntsPass
: public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
public:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 90a9674163f6b..8225323ebec14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -995,7 +995,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16);
- getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
+ // An alloca in the VGPR ("as memory") address space is custom-lowered to the
+ // address it was allocated; see legalizeFrameIndexVGPR.
+ getActionDefinitionsBuilder(G_FRAME_INDEX)
+ .legalFor({PrivatePtr})
+ .customFor({VGPRPtr});
// If the amount is divergent, we have to do a wave reduction to get the
// maximum value, so this is expanded during RegBankSelect.
@@ -2376,6 +2380,8 @@ bool AMDGPULegalizerInfo::legalizeCustom(
switch (MI.getOpcode()) {
case TargetOpcode::G_ADDRSPACE_CAST:
return legalizeAddrSpaceCast(MI, MRI, B);
+ case TargetOpcode::G_FRAME_INDEX:
+ return legalizeFrameIndexVGPR(MI, B);
case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
return legalizeFroundeven(MI, MRI, B);
case TargetOpcode::G_FCEIL:
@@ -3530,6 +3536,22 @@ static LLT widenToNextPowerOf2(LLT Ty) {
/// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for
/// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2).
/// Parallels the SelectionDAG LowerLoadStoreVGPR.
+// The address of an object in the VGPR ("as memory") address space is where
+// AMDGPUPromoteAlloca placed it, recorded on the alloca. Parallels the
+// SelectionDAG SITargetLowering::lowerFrameIndex.
+bool AMDGPULegalizerInfo::legalizeFrameIndexVGPR(MachineInstr &MI,
+ MachineIRBuilder &B) const {
+ MachineFunction &MF = B.getMF();
+ int FI = MI.getOperand(1).getIndex();
+ const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
+ assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR);
+
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ B.buildConstant(MI.getOperand(0), MD.getAddress());
+ MI.eraseFromParent();
+ return true;
+}
+
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
@@ -8943,6 +8965,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MI.eraseFromParent();
return true;
}
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
+ unsigned Opc = IntrID == Intrinsic::amdgcn_vgpr_lifetime_start
+ ? AMDGPU::VGPR_LIFETIME_START
+ : AMDGPU::VGPR_LIFETIME_END;
+ B.buildInstr(Opc).cloneMemRefs(MI);
+ MI.eraseFromParent();
+ return true;
+ }
case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 30fd16930e6ae..1cc9119575731 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -41,6 +41,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo {
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
+ bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const;
+
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const;
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index b109d0f5b2389..c8af1830378dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -16,6 +16,8 @@
#include "AMDGPU.h"
#include "AMDGPUAsmPrinter.h"
#include "AMDGPUMachineFunctionInfo.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
#include "MCTargetDesc/AMDGPUInstPrinter.h"
#include "MCTargetDesc/AMDGPUMCExpr.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -435,8 +437,19 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
}
if (MI->isMetaInstruction()) {
- if (isVerbose())
+ if (isVerbose()) {
+ if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(MI)) {
+ const auto &MD =
+ AMDGPU::AllocatedVGPRsMetadata::get(Marker->getObject());
+ unsigned Begin = MD.getAddress() / 4;
+ unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4;
+ OutStreamer->emitRawComment(
+ Twine(" VGPR lifetime ") + (Marker->isStart() ? "start" : "end") +
+ ": v[" + Twine(Begin) + ":" + Twine(End) + "]");
+ return;
+ }
OutStreamer->emitRawComment(" meta instruction");
+ }
return;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
index e99594062871a..9577b56c3ca1e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h
@@ -16,6 +16,7 @@
#include "SIInstrInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/IR/Instructions.h"
namespace llvm {
namespace AMDGPUMI {
@@ -74,6 +75,25 @@ class VStoreIdxInst : public VLoadStoreIdxInst {
}
};
+// Wrapper for the markers bounding the lifetime of an object in the VGPR "as
+// memory" (address space 13) address space. The object is named by the memory
+// operand, which is the only thing these carry until
+// AMDGPUPrivateObjectVGPRs gives them the registers it occupies.
+class VGPRLifetimeInst : public MachineInstr {
+public:
+ bool isStart() const { return getOpcode() == AMDGPU::VGPR_LIFETIME_START; }
+
+ const AllocaInst &getObject() const {
+ return *cast<AllocaInst>((*memoperands_begin())->getValue());
+ }
+
+ static bool classof(const MachineInstr *MI) {
+ unsigned Opc = MI->getOpcode();
+ return Opc == AMDGPU::VGPR_LIFETIME_START ||
+ Opc == AMDGPU::VGPR_LIFETIME_END;
+ }
+};
+
} // end namespace AMDGPUMI
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index fd5c0f16a289e..862182324b0df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -31,6 +31,32 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
GV->getValueType());
}
+AllocatedVGPRsMetadata &AllocatedVGPRsMetadata::get(const AllocaInst &Alloca) {
+ return *cast<AllocatedVGPRsMetadata>(
+ Alloca.getMetadata("amdgpu.allocated.vgprs"));
+}
+
+unsigned AllocatedVGPRsMetadata::getAddress() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(0))->getValue())
+ ->getZExtValue();
+}
+
+unsigned AllocatedVGPRsMetadata::getSize() const {
+ return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(1))->getValue())
+ ->getZExtValue();
+}
+
+bool AllocatedVGPRsMetadata::classof(const MDNode *N) {
+ if (N->getNumOperands() != 2)
+ return false;
+ for (int I = 0; I != 2; ++I) {
+ auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(I));
+ if (!C || !isa<ConstantInt>(C->getValue()))
+ return false;
+ }
+ return true;
+}
+
void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) {
SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
Source.getAllMetadata(MD);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index f81b677051561..db04c254feaeb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -12,11 +12,13 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/IR/Metadata.h"
namespace llvm {
struct Align;
class AAResults;
+class AllocaInst;
class DataLayout;
class GlobalVariable;
class LoadInst;
@@ -106,6 +108,23 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA);
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA);
+/// Convenience wrapper for !amdgpu.allocated.vgprs metadata, which records
+/// where in the VGPR ("as memory") address space an alloca was placed.
+///
+/// NOTE: Using `isa` with this class is not very meaningful.
+class AllocatedVGPRsMetadata : public MDNode {
+public:
+ static AllocatedVGPRsMetadata &get(const AllocaInst &Alloca);
+
+ /// Get the indicated (byte) address for the VGPR allocation.
+ unsigned getAddress() const;
+
+ /// Get the size (in bytes) of the VGPR allocation.
+ unsigned getSize() const;
+
+ static bool classof(const MDNode *N);
+};
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index b0ecd93b77ae4..36955a9d1b0a9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -75,6 +75,7 @@ FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
AMDGPUUnifyDivergentExitNodesPass())
+FUNCTION_PASS("amdgpu-vgpr-allocate", AMDGPUVGPRAllocatePass(*this))
FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this)))
FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
@@ -127,6 +128,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMA
MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
+MACHINE_FUNCTION_PASS("amdgpu-private-object-vgprs", AMDGPUPrivateObjectVGPRsPass())
MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
new file mode 100644
index 0000000000000..b4e14795e0708
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
@@ -0,0 +1,340 @@
+//===- AMDGPUPrivateObjectVGPRs.cpp - Private object VGPRs ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Mark the physical VGPRs an object in the VGPR ("as memory") address space
+/// was allocated to as used, so that register allocation does not hand them to
+/// anything else where the object is live:
+///
+/// * Add implicit use/def operands to the VGPR_LIFETIME_{START,END} pseudos
+/// * Add the VGPRs to basic block live-ins
+///
+/// The allocation itself is recorded on the alloca as !amdgpu.allocated.vgprs
+/// by AMDGPUPromoteAlloca, and reaches this pass through the memory operand of
+/// the lifetime markers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
+#include "AMDGPUMemoryUtils.h"
+#include "GCNSubtarget.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-private-object-vgprs"
+
+using ObjectRegs = SmallVector<MCPhysReg, 50>;
+
+namespace {
+
+class AMDGPUPrivateObjectVGPRsImpl {
+public:
+ AMDGPUPrivateObjectVGPRsImpl(const MachineFunction &MF, LiveIntervals *LIS)
+ : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()), LIS(LIS) {}
+
+ bool run(MachineFunction &MF);
+
+private:
+ struct AllocaBBInfo {
+ const AllocaInst *Alloca = nullptr;
+ bool LiveIn = false;
+ bool Starts = false;
+ bool Ends = false;
+ };
+
+ ObjectRegs computeObjectRegs(const AllocaInst &Alloca) const;
+
+ const SIInstrInfo *TII;
+ LiveIntervals *LIS;
+
+ DenseMap<const AllocaInst *, std::pair<ObjectRegs, MachineMemOperand *>>
+ AllocaObjectRegs;
+};
+
+} // end anonymous namespace
+
+// The registers an object occupies: one per dword, starting at the register its
+// byte address in the address space names.
+ObjectRegs AMDGPUPrivateObjectVGPRsImpl::computeObjectRegs(
+ const AllocaInst &Alloca) const {
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(Alloca);
+ unsigned Offset = MD.getAddress();
+ assert(Offset % 4 == 0 && "Object is not dword aligned");
+
+ // An object that does not fill its last register still owns the whole of it.
+ unsigned BaseRegIdx = Offset / 4;
+ unsigned NumRegs = divideCeil(MD.getSize(), 4);
+
+ ObjectRegs Regs;
+ Regs.reserve(NumRegs);
+ for (unsigned I : seq(NumRegs))
+ Regs.push_back(AMDGPU::VGPR_32RegClass.getRegister(BaseRegIdx + I));
+
+ return Regs;
+}
+
+bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) {
+ // Sort basic blocks in reverse post-order for the live-out/live-in
+ // propagation.
+ DenseMap<MachineBasicBlock *, unsigned> BlockToIndex;
+ SmallVector<MachineBasicBlock *> IndexToBlock;
+ ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
+ for (MachineBasicBlock *MBB : RPOT) {
+ BlockToIndex[MBB] = IndexToBlock.size();
+ IndexToBlock.push_back(MBB);
+ }
+
+ // Fixed-point iteration to determine basic block live-ins.
+ //
+ // The first pass of the fixed-point iteration also scans instructions.
+ SmallVector<SmallVector<AllocaBBInfo>> BBInfos(IndexToBlock.size());
+ SmallBitVector Worklist(IndexToBlock.size());
+ bool Changed = false;
+
+ for (bool Dirty = true, FirstPass = true; Dirty; FirstPass = false) {
+ Dirty = false;
+
+ for (auto [MBBI, MBB] : enumerate(IndexToBlock)) {
+ auto &BBI = BBInfos[MBBI];
+
+ // During the first outer iteration, augment VGPR_LIFETIME_{START,END}
+ // with implicit operands and record the initial per-basic block
+ // information to compute live-ins.
+ if (FirstPass) {
+ for (MachineInstr &MI : *MBB) {
+ if (MI.getOpcode() != AMDGPU::VGPR_LIFETIME_START &&
+ MI.getOpcode() != AMDGPU::VGPR_LIFETIME_END)
+ continue;
+
+ bool IsStart = MI.getOpcode() == AMDGPU::VGPR_LIFETIME_START;
+ MachineMemOperand *MMO = *MI.memoperands_begin();
+ const auto *Alloca = cast<AllocaInst>(MMO->getValue());
+
+ auto ObjRegsIt = AllocaObjectRegs.find(Alloca);
+ if (ObjRegsIt == AllocaObjectRegs.end())
+ ObjRegsIt =
+ AllocaObjectRegs
+ .try_emplace(Alloca, computeObjectRegs(*Alloca), MMO)
+ .first;
+
+ // The object comes into existence at the start and dies at the end.
+ for (MCPhysReg Reg : ObjRegsIt->second.first)
+ MI.addOperand(MachineOperand::CreateReg(
+ Reg, /*isDef=*/IsStart, /*isImp=*/true, /*isKill=*/!IsStart));
+
+ auto It = find_if(BBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == Alloca;
+ });
+ if (It == BBI.end()) {
+ BBI.push_back({Alloca, false, false, false});
+ It = std::prev(BBI.end());
+ }
+ It->Starts = IsStart;
+ It->Ends = !IsStart;
+
+ Changed = true;
+ }
+ } else {
+ if (!Worklist[MBBI])
+ continue;
+ Worklist[MBBI] = false;
+ }
+
+ // Propagate live-outs into successors.
+ for (const auto &ABBI : BBI) {
+ if (!((ABBI.LiveIn && !ABBI.Ends) || ABBI.Starts))
+ continue;
+
+ const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first;
+ for (MachineBasicBlock *Succ : MBB->successors()) {
+ unsigned SuccI = BlockToIndex.at(Succ);
+ auto &SuccBBI = BBInfos[SuccI];
+ auto It = find_if(SuccBBI, [&](const AllocaBBInfo &Info) {
+ return Info.Alloca == ABBI.Alloca;
+ });
+
+ bool Update = false;
+ if (It == SuccBBI.end()) {
+ SuccBBI.push_back({ABBI.Alloca, true, false, false});
+ It = std::prev(SuccBBI.end());
+ Update = true;
+ } else if (!It->LiveIn) {
+ It->LiveIn = true;
+ Update = true;
+ }
+
+ if (!Update)
+ continue;
+
+ // We are live-out from the successor because of the newly found
+ // live-in. If the successor is earlier in RPOT, we will have to
+ // re-evaluate it on the next outer iteration.
+ if (!It->Starts && !It->Ends && SuccI < MBBI) {
+ Worklist[SuccI] = true;
+ Dirty = true;
+ }
+
+ for (MCPhysReg Reg : Regs)
+ Succ->addLiveIn(Reg);
+ }
+ }
+ }
+ }
+
+ // It is legal for the pre-isel LLVM IR to have a lifetime.start without a
+ // lifetime.end. Liveness analysis is strong enough to mark physical registers
+ // as unused immediately after VGPR_LIFETIME_START in this case.
+ //
+ // Add VGPR_LIFETIME_END instructions at the end of basic blocks that end the
+ // function.
+ for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) {
+ if (!MBB->succ_empty())
+ continue;
+
+ for (const auto &ABBI : BBInfos[BBIdx]) {
+ if (ABBI.Ends || (!ABBI.LiveIn && !ABBI.Starts))
+ continue;
+
+ // There may be a COPY to a conflicting physical VGPR before a function
+ // return, so put the end as late as possible: walk back over the
+ // instructions that cannot be observing the object. Anything that touches
+ // memory, calls, or has side effects ends the walk, since the object has
+ // to stay reserved across it.
+ MachineBasicBlock::iterator IP = MBB->getFirstTerminator();
+ while (IP != MBB->begin()) {
+ --IP;
+ if (IP->mayStore() || IP->mayLoad() || IP->isCall() ||
+ IP->hasUnmodeledSideEffects()) {
+ ++IP;
+ break;
+ }
+ }
+
+ const auto &[ObjRegs, MMO] = AllocaObjectRegs.at(ABBI.Alloca);
+ MachineInstr *MI =
+ BuildMI(*MBB, IP, {}, TII->get(AMDGPU::VGPR_LIFETIME_END))
+ .addMemOperand(MMO);
+ for (MCPhysReg Reg : ObjRegs)
+ MI->addOperand(MachineOperand::CreateReg(
+ Reg, /*isDef=*/false, /*isImp=*/true, /*isKill=*/true));
+
+ Changed = true;
+ }
+ }
+
+ // An object lives in caller-saved registers, so a callee is free to overwrite
+ // it. Being live across a call is diagnosed rather than left to read back
+ // whatever the callee happened to leave behind. One diagnostic per object:
+ // the rest of its calls would say the same thing.
+ SmallPtrSet<const AllocaInst *, 4> Diagnosed;
+ for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) {
+ for (const auto &ABBI : BBInfos[BBIdx]) {
+ if (Diagnosed.contains(ABBI.Alloca))
+ continue;
+
+ bool Live = ABBI.LiveIn;
+ for (const MachineInstr &MI : *MBB) {
+ if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(&MI)) {
+ if (&Marker->getObject() == ABBI.Alloca)
+ Live = Marker->isStart();
+ continue;
+ }
+ if (!Live || !MI.isCall())
+ continue;
+
+ const Function &F = MF.getFunction();
+ F.getContext().diagnose(DiagnosticInfoUnsupported(
+ F,
+ "object in the VGPR 'as memory' address space (13) is live across "
+ "a call",
+ MI.getDebugLoc()));
+ Diagnosed.insert(ABBI.Alloca);
+ break;
+ }
+ }
+ }
+
+ // Remove live ranges from LiveIntervals. They will be recalculated lazily.
+ if (LIS) {
+ for (const auto &[Alloca, RegsAndMMO] : AllocaObjectRegs) {
+ for (MCPhysReg Reg : RegsAndMMO.first)
+ LIS->removeAllRegUnitsForPhysReg(Reg);
+ }
+ }
+
+ return Changed;
+}
+
+namespace {
+
+class AMDGPUPrivateObjectVGPRsLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUPrivateObjectVGPRsLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ LiveIntervals *LIS = nullptr;
+ if (auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>())
+ LIS = &LISWrapper->getLIS();
+ return AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF);
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ // LiveVariables only tracks virtual registers and we only touch physical
+ // registers.
+ AU.addPreserved<LiveVariablesWrapperPass>();
+ AU.addPreserved<SlotIndexesWrapperPass>();
+ AU.addPreserved<LiveIntervalsWrapperPass>();
+ AU.addPreservedID(MachineLoopInfoID);
+ AU.addPreservedID(MachineDominatorsID);
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override {
+ return "AMDGPU Def/use private object VGPRs";
+ }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPUPrivateObjectVGPRsPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
+ if (!AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF))
+ return PreservedAnalyses::all();
+
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<LiveVariablesAnalysis>();
+ PA.preserve<SlotIndexesAnalysis>();
+ PA.preserve<LiveIntervalsAnalysis>();
+ return PA;
+}
+
+char AMDGPUPrivateObjectVGPRsLegacy::ID = 0;
+
+char &llvm::AMDGPUPrivateObjectVGPRsID = AMDGPUPrivateObjectVGPRsLegacy::ID;
+
+INITIALIZE_PASS(AMDGPUPrivateObjectVGPRsLegacy, DEBUG_TYPE,
+ "AMDGPU Def/use private object VGPRs", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 900e8688d7f8c..ea36fb00e8dac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -26,6 +26,7 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "AMDGPUMemoryUtils.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/STLExtras.h"
@@ -142,6 +143,9 @@ class AMDGPUPromoteAllocaImpl {
bool IsAMDGCN = false;
bool IsAMDHSA = false;
+ /// Next free byte offset in the VGPR ("as memory") address space.
+ unsigned AllocVGPROffset = 0;
+
std::pair<Value *, Value *> getLocalSizeYZ(IRBuilder<> &Builder);
Value *getWorkitemID(IRBuilder<> &Builder, unsigned N);
@@ -167,6 +171,8 @@ class AMDGPUPromoteAllocaImpl {
void
finishDeferredAllocaToLDSPromotion(SetVector<IntrinsicInst *> &DeferredIntrs);
+ void allocateVgprs(AllocaAnalysis &AA);
+
void scoreAlloca(AllocaAnalysis &AA) const;
void setFunctionLimits(const Function &F);
@@ -179,7 +185,11 @@ class AMDGPUPromoteAllocaImpl {
IsAMDHSA = TT.getOS() == Triple::AMDHSA;
}
- bool run(Function &F, bool PromoteToLDS);
+ /// IsLatePass is true if invoked as a codegen pass and false if invoked from
+ /// the optimization pipeline ("amdgpu-promote-alloca-to-vector" pass). NoOpt
+ /// indicates to perform only the codegen work that is strictly required for
+ /// functionality, which is allocating objects in the VGPR address space.
+ bool run(Function &F, bool IsLatePass, bool NoOpt = false);
};
// FIXME: This can create globals so should be a module pass.
@@ -187,7 +197,8 @@ class AMDGPUPromoteAlloca : public FunctionPass {
public:
static char ID;
- AMDGPUPromoteAlloca() : FunctionPass(ID) {}
+ explicit AMDGPUPromoteAlloca(CodeGenOptLevel OptLevel)
+ : FunctionPass(ID), NoOpt(OptLevel == CodeGenOptLevel::None) {}
bool runOnFunction(Function &F) override {
if (skipFunction(F))
@@ -196,17 +207,22 @@ class AMDGPUPromoteAlloca : public FunctionPass {
return AMDGPUPromoteAllocaImpl(
TPC->getTM<TargetMachine>(), *F.getParent(),
getAnalysis<LoopInfoWrapperPass>().getLoopInfo())
- .run(F, /*PromoteToLDS*/ true);
+ .run(F, /*IsLatePass=*/true, NoOpt);
return false;
}
- StringRef getPassName() const override { return "AMDGPU Promote Alloca"; }
+ StringRef getPassName() const override {
+ return NoOpt ? "AMDGPU VGPR Allocate" : "AMDGPU Promote Alloca";
+ }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
AU.addRequired<LoopInfoWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
+
+private:
+ bool NoOpt;
};
static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
@@ -251,7 +267,7 @@ PreservedAnalyses AMDGPUPromoteAllocaPass::run(Function &F,
FunctionAnalysisManager &AM) {
auto &LI = AM.getResult<LoopAnalysis>(F);
bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
- .run(F, /*PromoteToLDS=*/true);
+ .run(F, /*IsLatePass=*/true);
if (Changed) {
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
@@ -264,7 +280,7 @@ PreservedAnalyses
AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) {
auto &LI = AM.getResult<LoopAnalysis>(F);
bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
- .run(F, /*PromoteToLDS=*/false);
+ .run(F, /*IsLatePass=*/false);
if (Changed) {
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
@@ -273,8 +289,21 @@ AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) {
return PreservedAnalyses::all();
}
-FunctionPass *llvm::createAMDGPUPromoteAlloca() {
- return new AMDGPUPromoteAlloca();
+FunctionPass *llvm::createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel) {
+ return new AMDGPUPromoteAlloca(OptLevel);
+}
+
+PreservedAnalyses AMDGPUVGPRAllocatePass::run(Function &F,
+ FunctionAnalysisManager &AM) {
+ auto &LI = AM.getResult<LoopAnalysis>(F);
+ bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI)
+ .run(F, /*IsLatePass=*/true, /*NoOpt=*/true);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
}
bool AMDGPUPromoteAllocaImpl::collectAllocaUses(AllocaAnalysis &AA) const {
@@ -367,10 +396,16 @@ void AMDGPUPromoteAllocaImpl::setFunctionLimits(const Function &F) {
VGPRBudgetRatio = PromoteAllocaToVectorVGPRRatio;
}
-bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
- if (DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector)
+bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
+ assert((!NoOpt || IsLatePass) && "NoOpt only makes sense for the late pass");
+
+ // Without optimizations only the allocation below is done, so the promotion
+ // options do not get a say.
+ if (!NoOpt && DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector)
return false;
+ const bool PromoteToLDS = IsLatePass && !NoOpt;
+
bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
MaxVGPRs = IsAMDGCN ? getMaxVGPRs(CurrentLocalMemUsage, TM, F) : 128;
setFunctionLimits(F);
@@ -391,6 +426,21 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
LLVM_DEBUG(dbgs() << "Analyzing: " << *AI << '\n');
AllocaAnalysis AA{AI};
+
+ // An alloca that is already in the VGPR ("as memory") address space is
+ // not promoted, only given a place in that address space, which only the
+ // codegen pass does. It happens whether or not optimizations are enabled,
+ // since the address space cannot be used without it.
+ if (AI->getAddressSpace() == AMDGPUAS::VGPR) {
+ if (IsLatePass)
+ Allocas.push_back(std::move(AA));
+ continue;
+ }
+
+ // Everything below this point is an optimization.
+ if (NoOpt)
+ continue;
+
if (collectAllocaUses(AA)) {
analyzePromoteToVector(AA);
if (PromoteToLDS)
@@ -403,8 +453,15 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
}
}
- stable_sort(Allocas,
- [](const auto &A, const auto &B) { return A.Score > B.Score; });
+ stable_sort(Allocas, [](const auto &A, const auto &B) {
+ // Prioritize allocas that are already in the VGPR address space, since
+ // their allocation must not fail.
+ bool AIsVGPR = A.Alloca->getAddressSpace() == AMDGPUAS::VGPR;
+ bool BIsVGPR = B.Alloca->getAddressSpace() == AMDGPUAS::VGPR;
+ if (AIsVGPR != BIsVGPR)
+ return AIsVGPR;
+ return A.Score > B.Score;
+ });
// clang-format off
LLVM_DEBUG(
@@ -417,6 +474,17 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
bool Changed = false;
SetVector<IntrinsicInst *> DeferredIntrs;
for (AllocaAnalysis &AA : Allocas) {
+ if (AA.Alloca->getAddressSpace() == AMDGPUAS::VGPR) {
+ allocateVgprs(AA);
+ std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL);
+ assert(Size); // Expected to succeed on non-array alloca.
+ const unsigned AllocaCost = Size->getFixedValue() * 8;
+ // These registers are no longer available for vectorization.
+ VectorizationBudget -= std::min(VectorizationBudget, AllocaCost);
+ Changed = true;
+ continue;
+ }
+
if (AA.Vector.Ty) {
std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL);
assert(Size); // Expected to succeed on non-array alloca.
@@ -451,6 +519,66 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
return Changed;
}
+// Give an alloca in the VGPR ("as memory") address space a place in that
+// address space, recorded on it as !amdgpu.allocated.vgprs, and lifetime
+// markers the backend understands. AMDGPUPrivateObjectVGPRs turns the two into
+// defs and uses of the physical registers the object occupies, which is what
+// keeps register allocation off them while it is live.
+void AMDGPUPromoteAllocaImpl::allocateVgprs(AllocaAnalysis &AA) {
+ LLVMContext &Ctx = Mod.getContext();
+ const unsigned AllocaSize =
+ DL.getTypeAllocSize(AA.Alloca->getAllocatedType()).getFixedValue();
+
+ // The generic lifetime intrinsics do not survive into the backend, so use
+ // the address-space specific ones instead. An object with no lifetime start
+ // is live from its definition.
+ bool HaveLifetimeStart = false;
+ for (Use &U : AA.Alloca->uses()) {
+ auto *II = dyn_cast<IntrinsicInst>(U.getUser());
+ if (!II)
+ continue;
+ if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ HaveLifetimeStart = true;
+ II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+ &Mod, Intrinsic::amdgcn_vgpr_lifetime_start, AA.Alloca->getType()));
+ } else if (II->getIntrinsicID() == Intrinsic::lifetime_end) {
+ II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
+ &Mod, Intrinsic::amdgcn_vgpr_lifetime_end, AA.Alloca->getType()));
+ }
+ }
+
+ if (!HaveLifetimeStart) {
+ Instruction *IP = AA.Alloca->getNextNode();
+ while (isa<AllocaInst>(IP))
+ IP = IP->getNextNode();
+ IRBuilder<> B(IP);
+ B.SetCurrentDebugLocation(AA.Alloca->getDebugLoc());
+ B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_vgpr_lifetime_start,
+ AA.Alloca);
+ }
+
+ // Allocating twice would place a second object on top of this one, so an
+ // object that already has its place keeps it, and the next one is allocated
+ // after it. The markers above are still rewritten, since IR that carries the
+ // metadata need not carry them.
+ if (MDNode *MD = AA.Alloca->getMetadata("amdgpu.allocated.vgprs")) {
+ const auto &Allocated = *cast<AMDGPU::AllocatedVGPRsMetadata>(MD);
+ AllocVGPROffset = std::max(AllocVGPROffset,
+ Allocated.getAddress() + alignTo(AllocaSize, 4));
+ return;
+ }
+
+ // Find space for the alloca. Objects are packed in order at dword
+ // granularity, since a register cannot be shared between two of them.
+ Type *I32 = Type::getInt32Ty(Ctx);
+ AA.Alloca->setMetadata(
+ "amdgpu.allocated.vgprs",
+ MDNode::get(
+ Ctx, {ConstantAsMetadata::get(ConstantInt::get(I32, AllocVGPROffset)),
+ ConstantAsMetadata::get(ConstantInt::get(I32, AllocaSize))}));
+ AllocVGPROffset += alignTo(AllocaSize, 4);
+}
+
// Checks if the instruction I is a memset user of the alloca AI that we can
// deal with. Currently, only non-volatile memsets that affect the whole alloca
// are handled.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index ee10f323f8c3f..bedfa9bedb33b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -679,6 +679,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSILowerWWMCopiesLegacyPass(*PR);
initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
initializeAMDGPULowerIdxOpsLegacyPass(*PR);
+ initializeAMDGPUPrivateObjectVGPRsLegacyPass(*PR);
initializeSILowerSGPRSpillsLegacyPass(*PR);
initializeSIFixSGPRCopiesLegacyPass(*PR);
initializeSIFixVGPRCopiesLegacyPass(*PR);
@@ -1598,9 +1599,12 @@ void AMDGPUPassConfig::addIRPasses() {
addPass(createAtomicExpandLegacyPass());
- if (TM.getOptLevel() > CodeGenOptLevel::None) {
- addPass(createAMDGPUPromoteAlloca());
+ // With optimizations enabled, do the full promotion of allocas. Without
+ // optimizations, only allocate existing addrspace(13) allocas, which is
+ // needed for functionality rather than for performance.
+ addPass(createAMDGPUPromoteAlloca(TM.getOptLevel()));
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses();
@@ -1815,6 +1819,8 @@ void GCNPassConfig::addFastRegAlloc() {
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID);
+
insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID);
TargetPassConfig::addFastRegAlloc();
@@ -1847,6 +1853,8 @@ void GCNPassConfig::addOptimizedRegAlloc() {
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID);
+ insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID);
+
if (EnableRewritePartialRegUses)
insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID);
@@ -2387,8 +2395,13 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
addFunctionPass(AtomicExpandPass(TM), PMW);
- if (TM.getOptLevel() > CodeGenOptLevel::None) {
+ // See the comment on the legacy pipeline above.
+ if (TM.getOptLevel() > CodeGenOptLevel::None)
addFunctionPass(AMDGPUPromoteAllocaPass(TM), PMW);
+ else
+ addFunctionPass(AMDGPUVGPRAllocatePass(TM), PMW);
+
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses(PMW);
@@ -2554,6 +2567,8 @@ void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const {
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass());
+
insertPass<TwoAddressInstructionPass>(SIWholeQuadModePass());
return Base::addFastRegAlloc(PMW);
@@ -2616,6 +2631,8 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
// SI_ELSE will introduce a copy of the tied operand source after the else.
insertPass<PHIEliminationPass>(SILowerControlFlowPass());
+ insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass());
+
if (EnableRewritePartialRegUses)
insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass());
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 5c9f8a2063847..6853b3f47081c 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -92,6 +92,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUIGroupLP.cpp
AMDGPULowerIdxOps.cpp
AMDGPULowerVGPREncoding.cpp
+ AMDGPUPrivateObjectVGPRs.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
AMDGPUMIRFormatter.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index aaf1742cd0143..435c01a5270aa 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -297,6 +297,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
+
+ // Only to give an alloca in the VGPR ("as memory") address space the address
+ // it was allocated; every other frame index is left alone.
+ setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
setOperationAction(ISD::BlockAddress, {MVT::i32, MVT::i64}, Custom);
setOperationAction(ISD::ExternalSymbol, {MVT::i32, MVT::i64}, Custom);
@@ -1615,6 +1619,22 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Infos.push_back(Info);
return;
}
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ // A marker covers the whole object, whose size and placement within the
+ // VGPR ("as memory") address space AMDGPUPromoteAlloca recorded on the
+ // alloca. Carrying that as a memory operand is what lets
+ // AMDGPUPrivateObjectVGPRs find the registers the object occupies.
+ const auto *Alloca = cast<AllocaInst>(CI.getArgOperand(0));
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ Info.opc = ISD::INTRINSIC_VOID;
+ Info.memVT = EVT::getIntegerVT(CI.getContext(), MD.getSize() * 8);
+ Info.ptrVal = Alloca;
+ Info.flags = Flags | MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOStore;
+ Infos.push_back(Info);
+ return;
+ }
case Intrinsic::amdgcn_ds_append:
case Intrinsic::amdgcn_ds_consume: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
@@ -7645,6 +7665,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerBRCOND(Op, DAG);
case ISD::RETURNADDR:
return LowerRETURNADDR(Op, DAG);
+ case ISD::FrameIndex:
+ return lowerFrameIndex(Op, DAG);
case ISD::SPONENTRY:
return LowerSPONENTRY(Op, DAG);
case ISD::LOAD: {
@@ -10168,6 +10190,20 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
MachineMemOperand::MOInvariant);
}
+// The address of an object in the VGPR ("as memory") address space is where
+// AMDGPUPromoteAlloca placed it, recorded on the alloca. Every other frame
+// index keeps its default lowering.
+SDValue SITargetLowering::lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
+ MachineFunction &MF = DAG.getMachineFunction();
+ int FI = cast<FrameIndexSDNode>(Op)->getIndex();
+ const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
+ if (!Alloca || Alloca->getAddressSpace() != AMDGPUAS::VGPR)
+ return SDValue();
+
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca);
+ return DAG.getConstant(MD.getAddress(), SDLoc(Op), MVT::i32);
+}
+
SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
SelectionDAG &DAG) const {
// TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
@@ -12570,6 +12606,15 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
unsigned IntrinsicID = Op.getConstantOperandVal(1);
switch (IntrinsicID) {
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ case Intrinsic::amdgcn_vgpr_lifetime_end: {
+ unsigned Opcode = IntrinsicID == Intrinsic::amdgcn_vgpr_lifetime_start
+ ? AMDGPU::VGPR_LIFETIME_START
+ : AMDGPU::VGPR_LIFETIME_END;
+ MachineSDNode *Marker = DAG.getMachineNode(Opcode, DL, MVT::Other, Chain);
+ DAG.setNodeMemRefs(Marker, {cast<MemSDNode>(Op)->getMemOperand()});
+ return SDValue(Marker, 0);
+ }
case Intrinsic::amdgcn_exp_compr: {
SDValue Src0 = Op.getOperand(4);
SDValue Src1 = Op.getOperand(5);
@@ -20334,6 +20379,17 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
Info->limitOccupancy(MF);
+ // An object in the VGPR "as memory" address space lives in registers, and
+ // every reference to its frame index has been replaced by the address it was
+ // allocated, so it must not take up stack space as well.
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ for (int FI = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); FI != E;
+ ++FI) {
+ const AllocaInst *Alloca = MFI.getObjectAllocation(FI);
+ if (Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR)
+ MFI.RemoveStackObject(FI);
+ }
+
// Give a VGPR "as memory" indexed access its M0 operand, matching how
// AMDGPULowerVGPREncoding will expand it. Under the VGPR indexing mode the
// expansion emits an s_set_gpr_idx_on, which reads the index out of its SGPR
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 6e089224184be..51995a4558667 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -187,6 +187,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
SelectionDAG &DAG) const;
+ SDValue lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 8257f8b6088e3..8f7bce6ecfaba 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4129,13 +4129,16 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
return false;
// VGPR "as memory" indexed accesses only alias each other, and then only
- // when their [idx+offset, idx+offset+width) dword ranges overlap.
+ // when their [idx+offset, idx+offset+width) dword ranges overlap. The
+ // exception is a lifetime marker, which conceptually overwrites the whole
+ // object it covers and so may alias any access to it.
const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa);
const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb);
if (IsLdStIdxA || IsLdStIdxB) {
if (IsLdStIdxA && IsLdStIdxB)
return checkInstOffsetsDoNotOverlap(MIa, MIb);
- return true;
+ return !isa<AMDGPUMI::VGPRLifetimeInst>(MIa) &&
+ !isa<AMDGPUMI::VGPRLifetimeInst>(MIb);
}
if (isLDSDMA(MIa) || isLDSDMA(MIb))
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 881dad361ca38..5587818e283c3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1161,6 +1161,25 @@ foreach vt = !listconcat(
Reg512Types.types, Reg1024Types.types) in
defm : VRegIdxLoadStorePat<vt>;
+/// Mark the start and end of the lifetime of an object in the VGPR ("as
+/// memory") address space. AMDGPUPrivateObjectVGPRs gives these the physical
+/// VGPRs the object was allocated to, as an implicit def on the start and an
+/// implicit killed use on the end, which is what keeps register allocation off
+/// those registers while the object is live.
+///
+/// These instructions conceptually overwrite the referenced memory with
+/// "undef".
+def VGPR_LIFETIME_START : SPseudoInstSI<(outs), (ins)> {
+ let hasSideEffects = false;
+ let mayStore = true;
+ let isMeta = true;
+}
+def VGPR_LIFETIME_END : SPseudoInstSI<(outs), (ins)> {
+ let hasSideEffects = false;
+ let mayStore = true;
+ let isMeta = true;
+}
+
// Used for byte or short loads.
// It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for
// sign-extended loads.
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
new file mode 100644
index 0000000000000..5e85b719ebeeb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
@@ -0,0 +1,80 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+
+; The diagnostic is an error, but code is still produced: check that a
+; synthesized end goes after the call rather than before it, since an end placed
+; early would release the object's registers while it is still live.
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s 2>/dev/null | FileCheck %s --check-prefix=ASM
+
+; ASM-LABEL: call_in_exit_block:
+; ASM: ; VGPR lifetime start: v[0:3]
+; ASM: s_swappc_b64
+; ASM: ; VGPR lifetime end: v[0:3]
+
+; An object in the VGPR "as memory" address space (13) is allocated to
+; caller-saved registers, so a callee is free to overwrite it. Being live across
+; a call is therefore diagnosed rather than quietly reading back whatever the
+; callee left behind.
+
+declare void @extern_func()
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+
+; CHECK: error: {{.*}}in function across_call{{.*}}object in the VGPR 'as memory' address space (13) is live across a call
+define void @across_call(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void @extern_func()
+ %v = load volatile i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The lifetime ends before the call, so the registers are free by then.
+; CHECK-NOT: in function dies_before_call
+define void @dies_before_call(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load volatile i32, ptr addrspace(13) %p
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ call void @extern_func()
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Reachability, not textual order: the object is still live on the path through
+; the call.
+; CHECK: error: {{.*}}in function across_call_in_branch{{.*}}live across a call
+define void @across_call_in_branch(ptr addrspace(1) %out, i32 %i, i1 %c) {
+entry:
+ %obj = alloca [4 x i32], addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ br i1 %c, label %call, label %join
+
+call:
+ call void @extern_func()
+ br label %join
+
+join:
+ %v = load volatile i32, ptr addrspace(13) %p
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; An object with no lifetime end is live to the end of the function, so a call
+; in the exit block is across it too: the synthesized end goes after the call,
+; not before it.
+; CHECK: error: {{.*}}in function call_in_exit_block{{.*}}live across a call
+define void @call_in_exit_block(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void @extern_func()
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
new file mode 100644
index 0000000000000..652798f0278a8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-promote-alloca -o - %s | FileCheck %s
+
+; An alloca in the VGPR "as memory" address space (13) is not promoted but
+; allocated: AMDGPUPromoteAlloca gives it a place in that address space,
+; recorded as !amdgpu.allocated.vgprs {byte offset, byte size}, and lifetime
+; markers the backend turns into defs and uses of the registers it occupies.
+
+define amdgpu_kernel void @one_object(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @one_object(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [4 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr addrspace(13) [[OBJ]], i32 0, i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Objects are packed in order, at dword granularity, since a register cannot be
+; shared between two of them.
+define amdgpu_kernel void @packed_objects(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @packed_objects(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca i8, align 1, addrspace(13), !amdgpu.allocated.vgprs [[META1:![0-9]+]]
+; CHECK-NEXT: [[B:%.*]] = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META2:![0-9]+]]
+; CHECK-NEXT: [[C:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META3:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[C]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[B]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[A]])
+; CHECK-NEXT: [[VA:%.*]] = load i8, ptr addrspace(13) [[A]], align 1
+; CHECK-NEXT: [[VB:%.*]] = load i32, ptr addrspace(13) [[B]], align 4
+; CHECK-NEXT: [[VC:%.*]] = load i32, ptr addrspace(13) [[C]], align 4
+; CHECK-NEXT: [[EA:%.*]] = zext i8 [[VA]] to i32
+; CHECK-NEXT: [[S0:%.*]] = add i32 [[EA]], [[VB]]
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[S0]], [[VC]]
+; CHECK-NEXT: store i32 [[S1]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca i8, addrspace(13)
+ %b = alloca [2 x i32], addrspace(13)
+ %c = alloca i32, addrspace(13)
+ %va = load i8, ptr addrspace(13) %a
+ %vb = load i32, ptr addrspace(13) %b
+ %vc = load i32, ptr addrspace(13) %c
+ %ea = zext i8 %va to i32
+ %s0 = add i32 %ea, %vb
+ %s1 = add i32 %s0, %vc
+ store i32 %s1, ptr addrspace(1) %out
+ ret void
+}
+
+; Existing lifetime markers become the address space specific ones rather than
+; a synthesized start.
+define amdgpu_kernel void @existing_lifetimes(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @existing_lifetimes(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META4:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ ret void
+}
+
+; Running again must not allocate a second place on top of the first, and must
+; not add a second marker.
+define amdgpu_kernel void @already_allocated(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @already_allocated(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META5:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !0
+ call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Metadata without the address-space specific markers: the place is kept, but
+; the generic markers still have to be rewritten, or nothing reserves the
+; registers in the backend.
+define amdgpu_kernel void @allocated_but_generic_markers(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @allocated_but_generic_markers(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META6:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: ret void
+;
+ %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !1
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ ret void
+}
+
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+declare void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) nocapture)
+
+!0 = !{i32 8, i32 4}
+!1 = !{i32 32, i32 4}
+;.
+; CHECK: [[META0]] = !{i32 0, i32 16}
+; CHECK: [[META1]] = !{i32 0, i32 1}
+; CHECK: [[META2]] = !{i32 4, i32 8}
+; CHECK: [[META3]] = !{i32 12, i32 4}
+; CHECK: [[META4]] = !{i32 0, i32 4}
+; CHECK: [[META5]] = !{i32 8, i32 4}
+; CHECK: [[META6]] = !{i32 32, i32 4}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
new file mode 100644
index 0000000000000..f1b9b8d378cdb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL
+; Allocating the object is needed for functionality, not performance, so it
+; happens without optimizations too.
+; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT
+; RUN: llc -O0 -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT
+
+; NOOPT: avoids_object_registers:
+; NOOPT: ; VGPR lifetime start: v[0:1]
+; NOOPT: v_movrel{{[sd]}}_b32
+; NOOPT: ; VGPR lifetime end: v[0:1]
+; NOOPT: ScratchSize: 0
+
+; The registers an object in the VGPR "as memory" address space (13) was
+; allocated must not be handed to anything else while it is live. The lifetime
+; markers carry them as an implicit def and an implicit killed use, so the
+; register allocator sees the object the same way it sees any other value.
+
+; The object occupies v[0:1], so the values live across it are allocated
+; elsewhere.
+define amdgpu_kernel void @avoids_object_registers(ptr addrspace(1) %out, i32 %i) {
+; GFX12-SDAG-LABEL: avoids_object_registers:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:1]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v2
+; GFX12-SDAG-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:1]
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, 7, v2
+; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: avoids_object_registers:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:1]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v2
+; GFX12-GISEL-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v4
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s4, v5
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:1]
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, s2, v2
+; GFX12-GISEL-NEXT: global_store_b32 v3, v2, s[0:1]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca [2 x i32], addrspace(13)
+ %p = getelementptr [2 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %a = load volatile i32, ptr addrspace(1) %out
+ %b = load volatile i32, ptr addrspace(1) %out
+ %c = load volatile i32, ptr addrspace(1) %out
+ %v = load i32, ptr addrspace(13) %p
+ %s0 = add i32 %a, %b
+ %s1 = add i32 %s0, %c
+ %s2 = add i32 %s1, %v
+ store i32 %s2, ptr addrspace(1) %out
+ ret void
+}
+
+; The object lives in registers, so it takes no stack space.
+define amdgpu_kernel void @no_scratch(ptr addrspace(1) %out, i32 %i) {
+; GFX12-SDAG-LABEL: no_scratch:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 7
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:3]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT: global_store_b32 v4, v5, s[0:1]
+; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v5
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:3]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: no_scratch:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v4, 7 :: v_dual_mov_b32 v5, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:3]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_bfe_u32 m0, s2, 0x1e0000
+; GFX12-GISEL-NEXT: global_store_b32 v5, v4, s[0:1]
+; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v4
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:3]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca [4 x i32], addrspace(13)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Registers are only unavailable where the object is live: after its end they
+; can be used again.
+define amdgpu_kernel void @registers_reused_after_end(ptr addrspace(1) %out) {
+; GFX12-SDAG-LABEL: registers_reused_after_end:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-SDAG-NEXT: s_mov_b32 m0, 0
+; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:0]
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:0]
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: global_store_b32 v1, v2, s[0:1]
+; GFX12-SDAG-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: registers_reused_after_end:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_mov_b32 m0, 0
+; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:0]
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0
+; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:0]
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: global_store_b32 v1, v2, s[0:1]
+; GFX12-GISEL-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-GISEL-NEXT: s_endpgm
+ %obj = alloca i32, addrspace(13)
+ call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj)
+ %v = load i32, ptr addrspace(13) %obj
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ %a = load volatile i32, ptr addrspace(1) %out
+ %b = load volatile i32, ptr addrspace(1) %out
+ %s = add i32 %a, %b
+ store i32 %s, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-private-object-vgprs.mir b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-private-object-vgprs.mir
new file mode 100644
index 0000000000000..b564f9494ff7e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-private-object-vgprs.mir
@@ -0,0 +1,106 @@
+# RUN: llc -mtriple=amdgpu12.00-- -verify-machineinstrs -run-pass=amdgpu-private-object-vgprs -o - %s | FileCheck %s
+
+# AMDGPUPrivateObjectVGPRs in isolation. The pass reads the object's placement
+# from the !amdgpu.allocated.vgprs metadata on the alloca named by each marker's
+# memory operand, and turns that into register operands the allocator can see:
+# an implicit def on the start, an implicit killed use on the end, plus live-ins
+# for the blocks in between.
+
+--- |
+ define amdgpu_kernel void @operands_on_the_markers(ptr addrspace(1) %out) {
+ %obj = alloca [4 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs !0
+ ret void
+ }
+
+ define amdgpu_kernel void @live_through_a_block(ptr addrspace(1) %out) {
+ %obj = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs !1
+ ret void
+ }
+
+ define amdgpu_kernel void @end_synthesized_in_exit_block(ptr addrspace(1) %out) {
+ %obj = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs !1
+ ret void
+ }
+
+ define amdgpu_kernel void @second_object_above_the_first(ptr addrspace(1) %out) {
+ %a = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs !1
+ %b = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs !2
+ ret void
+ }
+
+ !0 = !{i32 0, i32 16}
+ !1 = !{i32 0, i32 8}
+ !2 = !{i32 8, i32 8}
+...
+---
+# A four-dword object at byte 0 occupies v[0:3]: defined at the start, killed at
+# the end.
+name: operands_on_the_markers
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: operands_on_the_markers
+ ; CHECK: VGPR_LIFETIME_START implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3 :: (store (s128) into %ir.obj, addrspace 13)
+ ; CHECK-NEXT: VGPR_LIFETIME_END implicit killed $vgpr0, implicit killed $vgpr1, implicit killed $vgpr2, implicit killed $vgpr3 :: (store (s128) into %ir.obj, addrspace 13)
+ ; CHECK-NEXT: S_ENDPGM 0
+ VGPR_LIFETIME_START :: (store (s128) into %ir.obj, addrspace 13)
+ VGPR_LIFETIME_END :: (store (s128) into %ir.obj, addrspace 13)
+ S_ENDPGM 0
+...
+---
+# The object is live out of bb.0 and dies in bb.2, so both bb.1 and bb.2 must
+# have its registers as live-ins even though neither mentions it.
+name: live_through_a_block
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: live_through_a_block
+ ; CHECK: bb.0:
+ ; CHECK: VGPR_LIFETIME_START implicit-def $vgpr0, implicit-def $vgpr1
+ ; CHECK: bb.1:
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK: bb.2:
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1
+ ; CHECK: VGPR_LIFETIME_END implicit killed $vgpr0, implicit killed $vgpr1
+ bb.0:
+ successors: %bb.1
+ VGPR_LIFETIME_START :: (store (s64) into %ir.obj, addrspace 13)
+
+ bb.1:
+ successors: %bb.2
+
+ bb.2:
+ VGPR_LIFETIME_END :: (store (s64) into %ir.obj, addrspace 13)
+ S_ENDPGM 0
+...
+---
+# It is legal for the IR to have a start with no end. The pass then synthesizes
+# one in each exit block, as late as it can go.
+name: end_synthesized_in_exit_block
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: end_synthesized_in_exit_block
+ ; CHECK: VGPR_LIFETIME_START implicit-def $vgpr0, implicit-def $vgpr1
+ ; CHECK-NEXT: VGPR_LIFETIME_END implicit killed $vgpr0, implicit killed $vgpr1
+ ; CHECK-NEXT: S_ENDPGM 0
+ VGPR_LIFETIME_START :: (store (s64) into %ir.obj, addrspace 13)
+ S_ENDPGM 0
+...
+---
+# Two objects are packed one above the other, so the second one's registers
+# start where the first one's end.
+name: second_object_above_the_first
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: second_object_above_the_first
+ ; CHECK: VGPR_LIFETIME_START implicit-def $vgpr0, implicit-def $vgpr1 :: (store (s64) into %ir.a, addrspace 13)
+ ; CHECK-NEXT: VGPR_LIFETIME_START implicit-def $vgpr2, implicit-def $vgpr3 :: (store (s64) into %ir.b, addrspace 13)
+ ; CHECK-NEXT: VGPR_LIFETIME_END implicit killed $vgpr2, implicit killed $vgpr3 :: (store (s64) into %ir.b, addrspace 13)
+ ; CHECK-NEXT: VGPR_LIFETIME_END implicit killed $vgpr0, implicit killed $vgpr1 :: (store (s64) into %ir.a, addrspace 13)
+ VGPR_LIFETIME_START :: (store (s64) into %ir.a, addrspace 13)
+ VGPR_LIFETIME_START :: (store (s64) into %ir.b, addrspace 13)
+ VGPR_LIFETIME_END :: (store (s64) into %ir.b, addrspace 13)
+ VGPR_LIFETIME_END :: (store (s64) into %ir.a, addrspace 13)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index ba683ded7e575..de2b45e2c5aa0 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -29,6 +29,7 @@
; GCN-O0-NEXT: amdgpu-lower-module-lds
; GCN-O0-NEXT: function
; GCN-O0-NEXT: atomic-expand
+; GCN-O0-NEXT: amdgpu-vgpr-allocate
; GCN-O0-NEXT: verify
; GCN-O0-NEXT: unreachableblockelim
; GCN-O0-NEXT: ee-instrument<post-inline>
@@ -71,6 +72,7 @@
; GCN-O0-NEXT: amdgpu-lower-idx-ops
; GCN-O0-NEXT: phi-node-elimination
; GCN-O0-NEXT: si-lower-control-flow
+; GCN-O0-NEXT: amdgpu-private-object-vgprs
; GCN-O0-NEXT: two-address-instruction
; GCN-O0-NEXT: si-wqm
; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg
@@ -228,6 +230,7 @@
; GCN-O2-NEXT: require<machine-loops>
; GCN-O2-NEXT: phi-node-elimination
; GCN-O2-NEXT: si-lower-control-flow
+; GCN-O2-NEXT: amdgpu-private-object-vgprs
; GCN-O2-NEXT: two-address-instruction
; GCN-O2-NEXT: register-coalescer
; GCN-O2-NEXT: rename-independent-subregs
@@ -416,6 +419,7 @@
; GCN-O3-NEXT: require<machine-loops>
; GCN-O3-NEXT: phi-node-elimination
; GCN-O3-NEXT: si-lower-control-flow
+; GCN-O3-NEXT: amdgpu-private-object-vgprs
; GCN-O3-NEXT: two-address-instruction
; GCN-O3-NEXT: register-coalescer
; GCN-O3-NEXT: rename-independent-subregs
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 19d8cbae5c1e8..8d64b42b08f39 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -49,11 +49,13 @@
; GCN-O0-NEXT: Lower uses of LDS variables from non-kernel functions
; GCN-O0-NEXT: FunctionPass Manager
; GCN-O0-NEXT: Expand Atomic instructions
+; GCN-O0-NEXT: Dominator Tree Construction
+; GCN-O0-NEXT: Natural Loop Information
+; GCN-O0-NEXT: AMDGPU VGPR Allocate
; GCN-O0-NEXT: Remove unreachable blocks from the CFG
; GCN-O0-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; GCN-O0-NEXT: Scalarize Masked Memory Intrinsics
; GCN-O0-NEXT: Expand reduction intrinsics
-; GCN-O0-NEXT: Dominator Tree Construction
; GCN-O0-NEXT: AMDGPU Lower Kernel Arguments
; GCN-O0-NEXT: Lower buffer fat pointer operations to buffer resources
; GCN-O0-NEXT: FunctionPass Manager
@@ -116,6 +118,7 @@
; GCN-O0-NEXT: AMDGPU Lower Idx Ops
; GCN-O0-NEXT: Eliminate PHI nodes for register allocation
; GCN-O0-NEXT: SI Lower control flow pseudo instructions
+; GCN-O0-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O0-NEXT: Two-Address instruction pass
; GCN-O0-NEXT: MachineDominator Tree Construction
; GCN-O0-NEXT: Slot index numbering
@@ -367,6 +370,7 @@
; GCN-O1-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O1-NEXT: Two-Address instruction pass
; GCN-O1-NEXT: Slot index numbering
; GCN-O1-NEXT: Live Interval Analysis
@@ -696,6 +700,7 @@
; GCN-O1-OPTS-NEXT: SI Optimize VGPR LiveRange
; GCN-O1-OPTS-NEXT: Eliminate PHI nodes for register allocation
; GCN-O1-OPTS-NEXT: SI Lower control flow pseudo instructions
+; GCN-O1-OPTS-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O1-OPTS-NEXT: Two-Address instruction pass
; GCN-O1-OPTS-NEXT: Slot index numbering
; GCN-O1-OPTS-NEXT: Live Interval Analysis
@@ -1030,6 +1035,7 @@
; GCN-O2-NEXT: SI Optimize VGPR LiveRange
; GCN-O2-NEXT: Eliminate PHI nodes for register allocation
; GCN-O2-NEXT: SI Lower control flow pseudo instructions
+; GCN-O2-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O2-NEXT: Two-Address instruction pass
; GCN-O2-NEXT: Slot index numbering
; GCN-O2-NEXT: Live Interval Analysis
@@ -1380,6 +1386,7 @@
; GCN-O3-NEXT: SI Optimize VGPR LiveRange
; GCN-O3-NEXT: Eliminate PHI nodes for register allocation
; GCN-O3-NEXT: SI Lower control flow pseudo instructions
+; GCN-O3-NEXT: AMDGPU Def/use private object VGPRs
; GCN-O3-NEXT: Two-Address instruction pass
; GCN-O3-NEXT: Slot index numbering
; GCN-O3-NEXT: Live Interval Analysis
>From 334a7cdd9e67ee883345015e1bb9ffb8fdea84f6 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 11 Aug 2026 19:58:41 +0200
Subject: [PATCH 2/3] Diagnose inline asm that clobbers a VGPR-memory object's
registers
---
.../AMDGPU/AMDGPUPrivateObjectVGPRs.cpp | 36 ++++++-
.../AddressSpaceVGPR/as-vgpr-asm-clobber.ll | 93 +++++++++++++++++++
2 files changed, 124 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-asm-clobber.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
index b4e14795e0708..2dadcfe9ac124 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp
@@ -242,8 +242,11 @@ bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) {
// An object lives in caller-saved registers, so a callee is free to overwrite
// it. Being live across a call is diagnosed rather than left to read back
- // whatever the callee happened to leave behind. One diagnostic per object:
- // the rest of its calls would say the same thing.
+ // whatever the callee happened to leave behind. Inline asm is diagnosed too,
+ // but only when it clobbers registers the object occupies: unlike a call it
+ // names registers directly, so the liveness above cannot keep it away from
+ // them. One diagnostic per object: the rest would say the same thing.
+ const SIRegisterInfo &TRI = TII->getRegisterInfo();
SmallPtrSet<const AllocaInst *, 4> Diagnosed;
for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) {
for (const auto &ABBI : BBInfos[BBIdx]) {
@@ -257,14 +260,37 @@ bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) {
Live = Marker->isStart();
continue;
}
- if (!Live || !MI.isCall())
+ if (!Live)
+ continue;
+
+ std::string What;
+ if (MI.isCall()) {
+ What = "is live across a call";
+ } else if (MI.isInlineAsm()) {
+ const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first;
+ if (any_of(Regs, [&](MCPhysReg Reg) {
+ return MI.modifiesRegister(Reg, &TRI);
+ })) {
+ // Name the registers the object occupies. Unlike a callee, which
+ // may write any caller-saved register, the asm names a fixed set,
+ // so seeing the two side by side is what makes this fixable: either
+ // the asm gives those registers up, or the object is placed
+ // elsewhere.
+ const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*ABBI.Alloca);
+ unsigned Begin = MD.getAddress() / 4;
+ unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4;
+ What = ("at v[" + Twine(Begin) + ":" + Twine(End) +
+ "] is clobbered by inline asm")
+ .str();
+ }
+ }
+ if (What.empty())
continue;
const Function &F = MF.getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
- "object in the VGPR 'as memory' address space (13) is live across "
- "a call",
+ Twine("object in the VGPR 'as memory' address space (13) ") + What,
MI.getDebugLoc()));
Diagnosed.insert(ABBI.Alloca);
break;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-asm-clobber.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-asm-clobber.ll
new file mode 100644
index 0000000000000..84d555e67458e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-asm-clobber.ll
@@ -0,0 +1,93 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s
+
+; An object in the VGPR "as memory" address space (13) occupies the registers
+; its address names, and the liveness the backend builds keeps register
+; allocation off them. Inline asm is not bound by that: it names physical
+; registers directly. So an object live across asm that clobbers the registers
+; it occupies would quietly read back whatever the asm left, and is diagnosed.
+;
+; Unlike a call, which is rejected outright because a callee may write any
+; caller-saved register, asm is only rejected when it actually names registers
+; the object occupies.
+
+declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+
+; The object is placed at v[0:7], which is exactly what the asm clobbers. The
+; diagnostic names the registers it occupies, since that is what the asm's
+; clobber list has to be compared against.
+; CHECK: error: {{.*}}in function asm_clobbers_object{{.*}}object in the VGPR 'as memory' address space (13) at v[0:7] is clobbered by inline asm
+define amdgpu_kernel void @asm_clobbers_object(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13)
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void asm sideeffect "; clobber", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"()
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The same asm, with the object placed elsewhere: giving it an address that puts
+; it at v[40:47] resolves the conflict, which is one of the two ways out of the
+; diagnostic above.
+; CHECK-NOT: in function asm_object_moved_away
+define amdgpu_kernel void @asm_object_moved_away(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13), !amdgpu.allocated.vgprs !0
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void asm sideeffect "; clobber", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"()
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 160, i32 32}
+
+; Registers the object does not occupy are none of its business.
+; CHECK-NOT: in function asm_unrelated_vgpr
+define amdgpu_kernel void @asm_unrelated_vgpr(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13)
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void asm sideeffect "; nothing", "~{v100},~{v101}"()
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The object never lives in scalar registers.
+; CHECK-NOT: in function asm_sgpr_only
+define amdgpu_kernel void @asm_sgpr_only(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13)
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void asm sideeffect "; nothing", "~{s40},~{s41}"()
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Asm that clobbers nothing cannot disturb the object.
+; CHECK-NOT: in function asm_no_clobber
+define amdgpu_kernel void @asm_no_clobber(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13)
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ call void asm sideeffect "s_nop 0", ""()
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The lifetime ends first, so the registers are free by the time the asm runs.
+; CHECK-NOT: in function asm_after_lifetime
+define amdgpu_kernel void @asm_after_lifetime(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [8 x i32], addrspace(13)
+ %p = getelementptr [8 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load i32, ptr addrspace(13) %p
+ store i32 %v, ptr addrspace(1) %out
+ call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj)
+ call void asm sideeffect "; clobber", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"()
+ ret void
+}
>From e69d7339da8c33924520d3bf91f2ca0fb669e99d Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 11 Aug 2026 21:20:01 +0200
Subject: [PATCH 3/3] Do not add a second lifetime start when allocating VGPRs
again
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 20 +++++++++++++++---
.../AddressSpaceVGPR/as-vgpr-across-call.ll | 21 +++++++++++++++++++
.../AddressSpaceVGPR/as-vgpr-allocate.ll | 1 -
3 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index ea36fb00e8dac..52401ad2e6610 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -532,18 +532,32 @@ void AMDGPUPromoteAllocaImpl::allocateVgprs(AllocaAnalysis &AA) {
// The generic lifetime intrinsics do not survive into the backend, so use
// the address-space specific ones instead. An object with no lifetime start
// is live from its definition.
+ //
+ // A marker that is already the address-space specific one counts as a start:
+ // this runs over IR it has converted before, since an object written in the
+ // address space to begin with reaches the backend that way, as does one an
+ // earlier run promoted. Adding a second start would move the start of the
+ // live range back to the alloca and keep the object live from there.
bool HaveLifetimeStart = false;
for (Use &U : AA.Alloca->uses()) {
auto *II = dyn_cast<IntrinsicInst>(U.getUser());
if (!II)
continue;
- if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
- HaveLifetimeStart = true;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::lifetime_start:
II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
&Mod, Intrinsic::amdgcn_vgpr_lifetime_start, AA.Alloca->getType()));
- } else if (II->getIntrinsicID() == Intrinsic::lifetime_end) {
+ HaveLifetimeStart = true;
+ break;
+ case Intrinsic::lifetime_end:
II->setCalledFunction(Intrinsic::getOrInsertDeclaration(
&Mod, Intrinsic::amdgcn_vgpr_lifetime_end, AA.Alloca->getType()));
+ break;
+ case Intrinsic::amdgcn_vgpr_lifetime_start:
+ HaveLifetimeStart = true;
+ break;
+ default:
+ break;
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
index 5e85b719ebeeb..5272f0cee4acf 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll
@@ -19,6 +19,8 @@
declare void @extern_func()
declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture)
declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture)
+declare void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) nocapture)
+declare void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) nocapture)
; CHECK: error: {{.*}}in function across_call{{.*}}object in the VGPR 'as memory' address space (13) is live across a call
define void @across_call(ptr addrspace(1) %out, i32 %i) {
@@ -45,6 +47,25 @@ define void @dies_before_call(ptr addrspace(1) %out, i32 %i) {
ret void
}
+; The markers reaching the backend are the address-space specific ones, so an
+; object can arrive with its live range already marked out - that is how one
+; written in the address space to begin with gets here. Such a marker starts the
+; live range like any other: treating it as absent would start the range at the
+; alloca instead and make this call, which runs before the object is live, an
+; error.
+; CHECK-NOT: in function starts_after_call
+define void @starts_after_call(ptr addrspace(1) %out, i32 %i) {
+ %obj = alloca [4 x i32], addrspace(13)
+ call void @extern_func()
+ call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) %obj)
+ %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(13) %p
+ %v = load volatile i32, ptr addrspace(13) %p
+ call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) %obj)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
; Reachability, not textual order: the object is still live on the path through
; the call.
; CHECK: error: {{.*}}in function across_call_in_branch{{.*}}live across a call
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
index 652798f0278a8..497de3490f410 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll
@@ -83,7 +83,6 @@ define amdgpu_kernel void @already_allocated(ptr addrspace(1) %out) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META5:![0-9]+]]
; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
-; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4
; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
More information about the llvm-branch-commits
mailing list