[llvm-branch-commits] [llvm] [AMDGPU] Promote private objects into the VGPR address space via flag (PR #215282)
Gheorghe-Teodor Bercea via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Aug 25 12:19:28 PDT 2026
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/215282
>From b2030aaf8e3f6b7259048e1b5af0c477fa3c57bc Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Mon, 10 Aug 2026 15:45:03 +0200
Subject: [PATCH 1/4] Promote private objects into the VGPR address space via
flag
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 36 +--
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 19 ++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 24 +-
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 141 ++++++++-
.../AddressSpaceVGPR/as-vgpr-promote.ll | 279 ++++++++++++++++++
5 files changed, 460 insertions(+), 39 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 8225323ebec14..75336ee639993 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -560,40 +560,6 @@ static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
-// Whether the VGPR ("as memory") load/store lowering handles a MemSize-bit
-// memory access producing/consuming a ValSize-bit value at the given alignment.
-// Whole-dword accesses (those with a matching V_LOAD_IDX/V_STORE_IDX pseudo)
-// are supported when dword aligned, as are 8-/16-bit accesses, including
-// extending loads into a 16- or 32-bit value.
-//
-// A sub-dword access is implemented as a bit-field extract from (or insert
-// into) the dword containing it, so it must not straddle a dword boundary. An
-// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
-// natural alignment covers both, and is what lets the bit offset within the
-// dword be computed from a possibly dynamic pointer.
-//
-// A whole-dword access addresses registers by the dword index pointer >> 2,
-// which discards the low two bits rather than accounting for them, so an
-// under-aligned one would silently access the containing dword instead of the
-// bytes asked for.
-static bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
- Align Alignment) {
- if (MemSize == 8 || MemSize == 16) {
- if (Alignment < Align(MemSize / 8))
- return false;
- if (ValSize == MemSize)
- return true;
- if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
- return true;
- return false;
- }
- if (MemSize != ValSize)
- return false;
- if (Alignment < Align(4))
- return false;
- return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
-}
-
/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
@@ -3574,7 +3540,7 @@ static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) {
// Dword-aligned whole-dword and naturally aligned 8-/16-bit accesses are
// implemented. Reject anything else with a diagnostic instead of failing to
// legalize.
- if (!isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
+ if (!AMDGPU::isVGPRLoadStoreSupported(MemSize, ValSize, MMO.getAlign())) {
const Function &F = B.getMF().getFunction();
F.getContext().diagnose(DiagnosticInfoUnsupported(
F,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 862182324b0df..d92ac3583f677 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -8,6 +8,7 @@
#include "AMDGPUMemoryUtils.h"
#include "AMDGPU.h"
+#include "AMDGPUMachineInstrs.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/Analysis/AliasAnalysis.h"
@@ -46,6 +47,24 @@ unsigned AllocatedVGPRsMetadata::getSize() const {
->getZExtValue();
}
+bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment) {
+ if (MemSize == 8 || MemSize == 16) {
+ if (Alignment < Align(MemSize / 8))
+ return false;
+ if (ValSize == MemSize)
+ return true;
+ if (ValSize > MemSize && (ValSize == 16 || ValSize == 32))
+ return true;
+ return false;
+ }
+ if (MemSize != ValSize)
+ return false;
+ if (Alignment < Align(4))
+ return false;
+ return AMDGPUMI::VLoadIdxInst::tryGetOpcodeForBitWidth(MemSize) != -1;
+}
+
bool AllocatedVGPRsMetadata::classof(const MDNode *N) {
if (N->getNumOperands() != 2)
return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index db04c254feaeb..e0c41cf99c5ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -13,10 +13,10 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/IR/Metadata.h"
+#include "llvm/Support/Alignment.h"
namespace llvm {
-struct Align;
class AAResults;
class AllocaInst;
class DataLayout;
@@ -125,6 +125,28 @@ class AllocatedVGPRsMetadata : public MDNode {
static bool classof(const MDNode *N);
};
+/// Whether the VGPR ("as memory") address space implements a \p MemSize-bit
+/// memory access producing/consuming a \p ValSize-bit value at the given
+/// alignment. Whole-dword accesses (those with a matching V_LOAD_IDX /
+/// V_STORE_IDX pseudo) are supported when dword aligned, as are 8-/16-bit
+/// accesses, including extending loads into a 16- or 32-bit value.
+///
+/// A sub-dword access is implemented as a bit-field extract from (or insert
+/// into) the dword containing it, so it must not straddle a dword boundary. An
+/// 8-bit access never can; a 16-bit one only if it is 2-byte aligned. Requiring
+/// natural alignment covers both, and is what lets the bit offset within the
+/// dword be computed from a possibly dynamic pointer.
+///
+/// A whole-dword access addresses registers by the dword index pointer >> 2,
+/// which discards the low two bits rather than accounting for them, so an
+/// under-aligned one would silently access the containing dword instead of the
+/// bytes asked for.
+///
+/// Lowering diagnoses an access this rejects, so anything deciding to put an
+/// object in this address space has to agree with it.
+bool isVGPRLoadStoreSupported(unsigned MemSize, unsigned ValSize,
+ Align Alignment);
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 13bb542a48176..6352a469e1a0a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -37,6 +37,7 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
@@ -87,6 +88,14 @@ static cl::opt<unsigned>
"when sorting profitable allocas"),
cl::init(4));
+// An object in the VGPR ("as memory") address space cannot be spilled, so one
+// that is live across a call has nowhere to go: see analyzePromoteToVGPR.
+// TODO: Enable by default once objects can survive a call.
+static cl::opt<bool>
+ EnablePromoteToVGPR("amdgpu-promote-private",
+ cl::desc("Enable promoting private objects into VGPRs"),
+ cl::init(false), cl::Hidden);
+
// We support vector indices of the form ((A * stride) >> shift) + B
// VarIndex is A, VarMul is stride, VarShift is shift and ConstIndex is B. All
// parts are optional.
@@ -121,6 +130,9 @@ struct AllocaAnalysis {
bool Enable = false;
SmallVector<User *> Worklist;
} LDS;
+ struct {
+ bool Enable = false;
+ } VGPR;
explicit AllocaAnalysis(AllocaInst *Alloca) : Alloca(Alloca) {}
};
@@ -165,6 +177,8 @@ class AMDGPUPromoteAllocaImpl {
FixedVectorType *getVectorTypeForAlloca(Type *AllocaTy) const;
void analyzePromoteToVector(AllocaAnalysis &AA) const;
void promoteAllocaToVector(AllocaAnalysis &AA);
+ void analyzePromoteToVGPR(AllocaAnalysis &AA) const;
+ void promoteAllocaToVGPR(AllocaAnalysis &AA);
void analyzePromoteToLDS(AllocaAnalysis &AA) const;
bool tryPromoteAllocaToLDS(AllocaAnalysis &AA, bool SufficientLDS,
SetVector<IntrinsicInst *> &DeferredIntrs);
@@ -405,6 +419,27 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
return false;
const bool PromoteToLDS = IsLatePass && !NoOpt;
+ bool PromoteToVGPR = EnablePromoteToVGPR && IsLatePass && !NoOpt;
+
+ // An object in the VGPR ("as memory") address space occupies fixed registers
+ // for the whole of its live range. Those registers are caller-saved and the
+ // object cannot be spilled, so one that is live across a call has nowhere to
+ // be, and AMDGPUPrivateObjectVGPRs diagnoses it. Promoting nothing in a
+ // function that makes a call keeps this from turning working code into an
+ // error.
+ //
+ // Intrinsics do not count: on this target they lower to instructions rather
+ // than to calls. Were one ever to lower to a call, the result would be that
+ // diagnostic rather than a wrong answer.
+ //
+ // TODO: This is conservative. Only a call the object is live across matters,
+ // and then only one that does not preserve the registers it occupies.
+ if (PromoteToVGPR) {
+ PromoteToVGPR = none_of(instructions(F), [](const Instruction &I) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ return CB && !isa<IntrinsicInst>(CB);
+ });
+ }
bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
MaxVGPRs = IsAMDGCN ? getMaxVGPRs(CurrentLocalMemUsage, TM, F) : 128;
@@ -443,9 +478,11 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
if (collectAllocaUses(AA)) {
analyzePromoteToVector(AA);
+ if (PromoteToVGPR)
+ analyzePromoteToVGPR(AA);
if (PromoteToLDS)
analyzePromoteToLDS(AA);
- if (AA.Vector.Ty || AA.LDS.Enable) {
+ if (AA.Vector.Ty || AA.LDS.Enable || AA.VGPR.Enable) {
scoreAlloca(AA);
Allocas.push_back(std::move(AA));
}
@@ -485,13 +522,19 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
continue;
}
- if (AA.Vector.Ty) {
+ // Vectorization and promotion into the VGPR address space both spend the
+ // same registers, so they draw on the same budget. Vectorization is
+ // preferred where an alloca qualifies for either.
+ if (AA.Vector.Ty || AA.VGPR.Enable) {
std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL);
assert(Size); // Expected to succeed on non-array alloca.
const unsigned AllocaCost = Size->getFixedValue() * 8;
// First, check if we have enough budget to vectorize this alloca.
if (AllocaCost <= VectorizationBudget) {
- promoteAllocaToVector(AA);
+ if (AA.Vector.Ty)
+ promoteAllocaToVector(AA);
+ else
+ promoteAllocaToVGPR(AA);
Changed = true;
assert((VectorizationBudget - AllocaCost) < VectorizationBudget &&
"Underflow!");
@@ -1360,6 +1403,98 @@ void AMDGPUPromoteAllocaImpl::promoteAllocaToVector(AllocaAnalysis &AA) {
AA.Alloca->eraseFromParent();
}
+// Decide whether an alloca can be moved into the VGPR ("as memory") address
+// space, where it lives in registers rather than in scratch and is reached with
+// an indexed register access instead of a load or store.
+void AMDGPUPromoteAllocaImpl::analyzePromoteToVGPR(AllocaAnalysis &AA) const {
+ if (!IsAMDGCN)
+ return;
+
+ const auto Reject = [&](const Instruction *Inst, Twine Msg) {
+ LLVM_DEBUG(dbgs() << " Cannot promote alloca to VGPRs: " << Msg << "\n"
+ << " " << *Inst << "\n");
+ };
+
+ for (Use *U : AA.Uses) {
+ Instruction *Inst = cast<Instruction>(U->getUser());
+
+ if (getLoadStorePointerOperand(Inst)) {
+ assert(!isa<StoreInst>(Inst) ||
+ U->getOperandNo() == StoreInst::getPointerOperandIndex());
+
+ bool IsSimple = isa<LoadInst>(Inst) ? cast<LoadInst>(Inst)->isSimple()
+ : cast<StoreInst>(Inst)->isSimple();
+ if (!IsSimple)
+ return Reject(Inst, "not a simple load or store");
+
+ // Promoting an access the lowering cannot implement would turn this into
+ // a diagnostic, so ask the lowering rather than guessing.
+ TypeSize AccessSize = DL.getTypeSizeInBits(getLoadStoreType(Inst));
+ if (AccessSize.isScalable())
+ return Reject(Inst, "scalable access");
+
+ // The value and memory sizes are the same here: an extending load of an
+ // object in private memory is a plain load followed by an extend.
+ unsigned Bits = AccessSize.getFixedValue();
+ Align Alignment = isa<LoadInst>(Inst) ? cast<LoadInst>(Inst)->getAlign()
+ : cast<StoreInst>(Inst)->getAlign();
+ if (!AMDGPU::isVGPRLoadStoreSupported(Bits, Bits, Alignment))
+ return Reject(Inst, "unsupported access size or alignment");
+
+ continue;
+ }
+
+ // These only compute addresses; collectAllocaUses has already established
+ // that a select or phi does not mix objects.
+ if (isa<GetElementPtrInst, SelectInst, PHINode>(Inst))
+ continue;
+
+ if (auto *MSI = dyn_cast<MemSetInst>(Inst)) {
+ if (!isSupportedMemset(MSI, AA.Alloca, DL))
+ return Reject(MSI, "cannot handle partial memset");
+ continue;
+ }
+
+ if (isa<MemTransferInst>(Inst))
+ return Reject(Inst, "cannot handle mem transfer");
+
+ if (auto *Intr = dyn_cast<IntrinsicInst>(Inst)) {
+ if (Intr->getIntrinsicID() == Intrinsic::objectsize)
+ continue;
+
+ if (isAssumeLikeIntrinsic(Inst)) {
+ if (!Inst->use_empty())
+ return Reject(Inst, "assume-like intrinsic cannot have any users");
+ continue;
+ }
+ }
+
+ // A comparison whose only purpose is to feed an assume.
+ if (isa<ICmpInst>(Inst) && all_of(Inst->users(), [](User *U) {
+ return isAssumeLikeIntrinsic(cast<Instruction>(U));
+ }))
+ continue;
+
+ return Reject(Inst, "unhandled alloca user");
+ }
+
+ AA.VGPR.Enable = true;
+}
+
+// Move the alloca into the VGPR ("as memory") address space. Pointers into it
+// are the same size in both address spaces, so the pointers derived from it
+// only need their type changed, and allocateVgprs then gives the object its
+// place in that address space.
+void AMDGPUPromoteAllocaImpl::promoteAllocaToVGPR(AllocaAnalysis &AA) {
+ LLVM_DEBUG(dbgs() << "Promoting alloca to VGPRs: " << *AA.Alloca << '\n');
+
+ Type *PtrTy = PointerType::get(Mod.getContext(), AMDGPUAS::VGPR);
+ for (Value *Ptr : AA.Pointers)
+ Ptr->mutateType(PtrTy);
+
+ allocateVgprs(AA);
+}
+
std::pair<Value *, Value *>
AMDGPUPromoteAllocaImpl::getLocalSizeYZ(IRBuilder<> &Builder) {
Function &F = *Builder.GetInsertBlock()->getParent();
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
new file mode 100644
index 0000000000000..c0b15b5f954e7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
@@ -0,0 +1,279 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgpu12.00-- -passes=amdgpu-promote-alloca \
+; RUN: -amdgpu-promote-private -o - %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu9.42-- -passes=amdgpu-promote-alloca \
+; RUN: -amdgpu-promote-private -o - %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu12.00-- -passes=amdgpu-promote-alloca -o - %s \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; A private alloca can be moved into the VGPR "as memory" address space (13),
+; where it lives in registers rather than in scratch. The pointers derived from
+; it change address space and the object is then allocated exactly as one
+; written in that address space to begin with.
+;
+; Promotion is behind -amdgpu-promote-private, so the OFF prefix checks that
+; nothing moves without it. It needs no target feature beyond what the address
+; space itself needs, so the two triples cover both the movrel and the
+; VGPR-index-mode lowerings.
+
+; The object is indexed at byte granularity, which vectorization cannot express,
+; so this reaches the VGPR path rather than being turned into a vector.
+define amdgpu_kernel void @promoted(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @promoted(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @promoted(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %v = load i32, ptr addrspace(5) %p, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Sub-dword accesses are supported, so a mix of widths does not prevent this.
+define amdgpu_kernel void @promoted_sub_dword(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @promoted_sub_dword(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(13) [[OBJ]], i32 0, i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr addrspace(13) [[Q]], align 1
+; CHECK-NEXT: [[Z:%.*]] = zext i8 [[V]] to i32
+; CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @promoted_sub_dword(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[OBJ]], i32 0, i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[Q:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load i8, ptr addrspace(5) [[Q]], align 1
+; OFF-NEXT: [[Z:%.*]] = zext i8 [[V]] to i32
+; OFF-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(5) %p
+ %q = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load i8, ptr addrspace(5) %q
+ %z = zext i8 %v to i32
+ store i32 %z, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @extern()
+
+; An object in this address space cannot be spilled and the registers it
+; occupies are caller-saved, so one live across a call has nowhere to be and the
+; backend rejects it. Promotion therefore declines rather than turning working
+; code into an error. This is conservative: the call need not be in the object's
+; live range for promotion to be refused.
+define amdgpu_kernel void @not_promoted_call(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @not_promoted_call(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: call void @extern()
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @not_promoted_call(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @extern()
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ call void @extern()
+ %v = load i32, ptr addrspace(5) %p, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Only accesses the backend implements: an under-aligned one is not one of them.
+define amdgpu_kernel void @not_promoted_misaligned(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @not_promoted_misaligned(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 1
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @not_promoted_misaligned(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 1
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load i32, ptr addrspace(5) %p, align 1
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; An access the lowering does not implement would only be diagnosed later, so it
+; is rejected here. This one is neither a whole dword nor 8 or 16 bits, despite
+; being sufficiently aligned.
+define amdgpu_kernel void @not_promoted_odd_size(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @not_promoted_odd_size(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load [3 x i8], ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: store [3 x i8] [[V]], ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @not_promoted_odd_size(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load [3 x i8], ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store [3 x i8] [[V]], ptr addrspace(1) [[OUT]], align 1
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load [3 x i8], ptr addrspace(5) %p, align 4
+ store [3 x i8] %v, ptr addrspace(1) %out
+ ret void
+}
+
+; A volatile access is not a plain indexed register access.
+define amdgpu_kernel void @not_promoted_volatile(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @not_promoted_volatile(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load volatile i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @not_promoted_volatile(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load volatile i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load volatile i32, ptr addrspace(5) %p, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+declare void @llvm.memcpy.p5.p5.i32(ptr addrspace(5), ptr addrspace(5), i32, i1)
+
+; A copy between two objects is not an indexed access either.
+define amdgpu_kernel void @not_promoted_memcpy(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @not_promoted_memcpy(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 1025) i32 @llvm.r600.read.local.size.y()
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 1025) i32 @llvm.r600.read.local.size.z()
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT: [[TMP5:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i32 [[TMP4]], [[TMP2]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [1024 x [8 x i32]], ptr addrspace(3) @not_promoted_memcpy.other, i32 0, i32 [[TMP10]]
+; CHECK-NEXT: call void @llvm.memcpy.p5.p3.i32(ptr addrspace(5) [[OBJ]], ptr addrspace(3) [[TMP11]], i32 32, i1 false)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @not_promoted_memcpy(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[TMP1:%.*]] = call range(i32 0, 1025) i32 @llvm.r600.read.local.size.y()
+; OFF-NEXT: [[TMP2:%.*]] = call range(i32 0, 1025) i32 @llvm.r600.read.local.size.z()
+; OFF-NEXT: [[TMP3:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; OFF-NEXT: [[TMP4:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; OFF-NEXT: [[TMP5:%.*]] = call range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+; OFF-NEXT: [[TMP6:%.*]] = mul nuw nsw i32 [[TMP1]], [[TMP2]]
+; OFF-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], [[TMP3]]
+; OFF-NEXT: [[TMP8:%.*]] = mul nuw nsw i32 [[TMP4]], [[TMP2]]
+; OFF-NEXT: [[TMP9:%.*]] = add i32 [[TMP7]], [[TMP8]]
+; OFF-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP5]]
+; OFF-NEXT: [[TMP11:%.*]] = getelementptr inbounds [1024 x [8 x i32]], ptr addrspace(3) @not_promoted_memcpy.other, i32 0, i32 [[TMP10]]
+; OFF-NEXT: call void @llvm.memcpy.p5.p3.i32(ptr addrspace(5) [[OBJ]], ptr addrspace(3) [[TMP11]], i32 32, i1 false)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %other = alloca [8 x i32], align 4, addrspace(5)
+ call void @llvm.memcpy.p5.p5.i32(ptr addrspace(5) %obj, ptr addrspace(5) %other,
+ i32 32, i1 false)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load i32, ptr addrspace(5) %p, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Where an alloca qualifies for both, vectorization wins: the two spend the same
+; registers, and a vector needs no indexed access to read an element.
+define amdgpu_kernel void @vector_preferred(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @vector_preferred(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 7, i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @vector_preferred(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = freeze <8 x i32> poison
+; OFF-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> [[OBJ]], i32 7, i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %obj, i32 0, i32 %i
+ store i32 7, ptr addrspace(5) %p
+ %v = load i32, ptr addrspace(5) %p
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+;.
+; CHECK: [[META0]] = !{i32 0, i32 32}
+;.
>From 0946bbe70469a9167e5937b69cba38a6c195203a Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 11 Aug 2026 18:37:27 +0200
Subject: [PATCH 2/4] Carry null operands and intrinsic names across when
promoting to VGPRs
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 44 +++++-
.../AddressSpaceVGPR/as-vgpr-promote.ll | 145 ++++++++++++++++++
2 files changed, 187 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 6352a469e1a0a..5a112a5b6c158 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -1484,14 +1484,54 @@ void AMDGPUPromoteAllocaImpl::analyzePromoteToVGPR(AllocaAnalysis &AA) const {
// Move the alloca into the VGPR ("as memory") address space. Pointers into it
// are the same size in both address spaces, so the pointers derived from it
// only need their type changed, and allocateVgprs then gives the object its
-// place in that address space.
+// place in that address space. This mirrors what the LDS promotion below does
+// to the same kind of closed set of derived pointers.
void AMDGPUPromoteAllocaImpl::promoteAllocaToVGPR(AllocaAnalysis &AA) {
LLVM_DEBUG(dbgs() << "Promoting alloca to VGPRs: " << *AA.Alloca << '\n');
Type *PtrTy = PointerType::get(Mod.getContext(), AMDGPUAS::VGPR);
- for (Value *Ptr : AA.Pointers)
+ for (Value *Ptr : AA.Pointers) {
Ptr->mutateType(PtrTy);
+ // A select or phi may pick between a pointer into the object and a null
+ // one, which collectAllocaUses allows. Changing the address space of the
+ // result leaves such a constant behind in the old one, so adjust it too.
+ if (auto *SI = dyn_cast<SelectInst>(Ptr)) {
+ for (unsigned I : {1, 2})
+ if (isa<ConstantPointerNull, ConstantAggregateZero>(SI->getOperand(I)))
+ SI->setOperand(I, Constant::getNullValue(PtrTy));
+ } else if (auto *Phi = dyn_cast<PHINode>(Ptr)) {
+ for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I)
+ if (isa<ConstantPointerNull, ConstantAggregateZero>(
+ Phi->getIncomingValue(I)))
+ Phi->setIncomingValue(I, Constant::getNullValue(PtrTy));
+ }
+ }
+
+ // An intrinsic overloaded on the pointer type still names the old address
+ // space in its mangled name, which no longer matches the argument it is being
+ // given. Rebuild those, letting the builder derive the name from the types.
+ // The lifetime markers are left alone: allocateVgprs replaces them with the
+ // address-space specific ones below.
+ SmallSetVector<IntrinsicInst *, 4> Rebuild;
+ for (Use *U : AA.Uses) {
+ auto *II = dyn_cast<IntrinsicInst>(U->getUser());
+ if (II && !II->isLifetimeStartOrEnd())
+ Rebuild.insert(II);
+ }
+
+ for (IntrinsicInst *II : Rebuild) {
+ IRBuilder<> B(II);
+ SmallVector<Value *> Args(II->args());
+ Value *New = B.CreateIntrinsic(II->getType(), II->getIntrinsicID(), Args);
+ if (auto *NewCall = dyn_cast<CallInst>(New)) {
+ NewCall->copyMetadata(*II);
+ NewCall->takeName(II);
+ }
+ II->replaceAllUsesWith(New);
+ II->eraseFromParent();
+ }
+
allocateVgprs(AA);
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
index c0b15b5f954e7..e79286a838b29 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
@@ -250,6 +250,151 @@ define amdgpu_kernel void @not_promoted_memcpy(ptr addrspace(1) %out, i32 %i) {
ret void
}
+; An intrinsic overloaded on the pointer type has the address space in its
+; mangled name, so moving the object has to rebuild the call, or the name no
+; longer describes the argument.
+define amdgpu_kernel void @memset_object(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @memset_object(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: call void @llvm.memset.p13.i32(ptr addrspace(13) [[OBJ]], i8 0, i32 32, i1 false)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @memset_object(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: call void @llvm.memset.p5.i32(ptr addrspace(5) [[OBJ]], i8 0, i32 32, i1 false)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ call void @llvm.memset.p5.i32(ptr addrspace(5) %obj, i8 0, i32 32, i1 false)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load i32, ptr addrspace(5) %p, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @objectsize_object(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @objectsize_object(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[S:%.*]] = call i64 @llvm.objectsize.i64.p13(ptr addrspace(13) [[OBJ]], i1 false, i1 false, i1 false)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[T:%.*]] = trunc i64 [[S]] to i32
+; CHECK-NEXT: [[R:%.*]] = add i32 [[V]], [[T]]
+; CHECK-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @objectsize_object(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[S:%.*]] = call i64 @llvm.objectsize.i64.p5(ptr addrspace(5) [[OBJ]], i1 false, i1 false, i1 false)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[T:%.*]] = trunc i64 [[S]] to i32
+; OFF-NEXT: [[R:%.*]] = add i32 [[V]], [[T]]
+; OFF-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %s = call i64 @llvm.objectsize.i64.p5(ptr addrspace(5) %obj, i1 false, i1 false, i1 false)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %v = load i32, ptr addrspace(5) %p, align 4
+ %t = trunc i64 %s to i32
+ %r = add i32 %v, %t
+ store i32 %r, ptr addrspace(1) %out
+ ret void
+}
+
+; A select or phi is allowed to pick between a pointer into the object and a
+; null one. Moving the result to another address space has to take the constant
+; with it, or the operands no longer agree and the IR is invalid.
+define amdgpu_kernel void @select_null(ptr addrspace(1) %out, i32 %i, i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @select_null(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr addrspace(13) [[P]], ptr addrspace(13) null
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[SEL]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[SEL]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @select_null(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr addrspace(5) [[P]], ptr addrspace(5) null
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[SEL]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[SEL]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ %sel = select i1 %c, ptr addrspace(5) %p, ptr addrspace(5) null
+ store i32 7, ptr addrspace(5) %sel, align 4
+ %v = load i32, ptr addrspace(5) %sel, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @phi_null(ptr addrspace(1) %out, i32 %i, i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @phi_null(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: br i1 [[C]], label %[[USE:.*]], label %[[OTHER:.*]]
+; CHECK: [[OTHER]]:
+; CHECK-NEXT: br label %[[USE]]
+; CHECK: [[USE]]:
+; CHECK-NEXT: [[PH:%.*]] = phi ptr addrspace(13) [ [[P]], %[[ENTRY]] ], [ null, %[[OTHER]] ]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[PH]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[PH]], align 4
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @phi_null(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; OFF-NEXT: [[ENTRY:.*]]:
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: br i1 [[C]], label %[[USE:.*]], label %[[OTHER:.*]]
+; OFF: [[OTHER]]:
+; OFF-NEXT: br label %[[USE]]
+; OFF: [[USE]]:
+; OFF-NEXT: [[PH:%.*]] = phi ptr addrspace(5) [ [[P]], %[[ENTRY]] ], [ null, %[[OTHER]] ]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[PH]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[PH]], align 4
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+entry:
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ br i1 %c, label %use, label %other
+other:
+ br label %use
+use:
+ %ph = phi ptr addrspace(5) [ %p, %entry ], [ null, %other ]
+ store i32 7, ptr addrspace(5) %ph, align 4
+ %v = load i32, ptr addrspace(5) %ph, align 4
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
; Where an alloca qualifies for both, vectorization wins: the two spend the same
; registers, and a vector needs no indexed access to read an element.
define amdgpu_kernel void @vector_preferred(ptr addrspace(1) %out, i32 %i) {
>From 09b8d253e24afe5614859bccabaa4d3ae2cba410 Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 11 Aug 2026 21:42:47 +0200
Subject: [PATCH 3/4] Refuse promotion only when the object is live across a
call
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 134 +++++++++++--
.../AddressSpaceVGPR/as-vgpr-promote.ll | 187 +++++++++++++++++-
2 files changed, 301 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 5a112a5b6c158..0a070e20950fa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -158,6 +158,10 @@ class AMDGPUPromoteAllocaImpl {
/// Next free byte offset in the VGPR ("as memory") address space.
unsigned AllocVGPROffset = 0;
+ /// Blocks holding a call an object in the VGPR address space could be live
+ /// across. Empty unless that promotion is enabled; see isLiveAcrossCall.
+ SmallPtrSet<const BasicBlock *, 8> CallBlocks;
+
std::pair<Value *, Value *> getLocalSizeYZ(IRBuilder<> &Builder);
Value *getWorkitemID(IRBuilder<> &Builder, unsigned N);
@@ -177,6 +181,7 @@ class AMDGPUPromoteAllocaImpl {
FixedVectorType *getVectorTypeForAlloca(Type *AllocaTy) const;
void analyzePromoteToVector(AllocaAnalysis &AA) const;
void promoteAllocaToVector(AllocaAnalysis &AA);
+ bool isLiveAcrossCall(const AllocaInst *AI) const;
void analyzePromoteToVGPR(AllocaAnalysis &AA) const;
void promoteAllocaToVGPR(AllocaAnalysis &AA);
void analyzePromoteToLDS(AllocaAnalysis &AA) const;
@@ -419,26 +424,22 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
return false;
const bool PromoteToLDS = IsLatePass && !NoOpt;
- bool PromoteToVGPR = EnablePromoteToVGPR && IsLatePass && !NoOpt;
-
- // An object in the VGPR ("as memory") address space occupies fixed registers
- // for the whole of its live range. Those registers are caller-saved and the
- // object cannot be spilled, so one that is live across a call has nowhere to
- // be, and AMDGPUPrivateObjectVGPRs diagnoses it. Promoting nothing in a
- // function that makes a call keeps this from turning working code into an
- // error.
+ const bool PromoteToVGPR = EnablePromoteToVGPR && IsLatePass && !NoOpt;
+
+ // Whether an object could be live across one of these is decided per alloca,
+ // in analyzePromoteToVGPR. Collected once here because the answer is a
+ // property of the function, not of the object.
//
// Intrinsics do not count: on this target they lower to instructions rather
- // than to calls. Were one ever to lower to a call, the result would be that
- // diagnostic rather than a wrong answer.
- //
- // TODO: This is conservative. Only a call the object is live across matters,
- // and then only one that does not preserve the registers it occupies.
+ // than to calls. Were one ever to lower to a call, the result would be the
+ // backend's diagnostic rather than a wrong answer.
+ CallBlocks.clear();
if (PromoteToVGPR) {
- PromoteToVGPR = none_of(instructions(F), [](const Instruction &I) {
+ for (const Instruction &I : instructions(F)) {
const auto *CB = dyn_cast<CallBase>(&I);
- return CB && !isa<IntrinsicInst>(CB);
- });
+ if (CB && !isa<IntrinsicInst>(CB))
+ CallBlocks.insert(I.getParent());
+ }
}
bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
@@ -1403,6 +1404,94 @@ void AMDGPUPromoteAllocaImpl::promoteAllocaToVector(AllocaAnalysis &AA) {
AA.Alloca->eraseFromParent();
}
+// Whether a call can execute while \p AI is live.
+//
+// An object in the VGPR ("as memory") address space occupies fixed registers
+// for the whole of its live range. Those registers are caller-saved and the
+// object cannot be spilled, so one that is live across a call has nowhere to
+// be, and AMDGPUPrivateObjectVGPRs diagnoses it. Declining to promote in that
+// case keeps this from turning a working program into an error.
+//
+// The live range is the one that pass will use, not the one the uses imply: an
+// object is live from its lifetime start - or from the alloca, if it has none,
+// since allocateVgprs then inserts a start there - until its lifetime end, or
+// the end of the function. An object with no use after a call is still live
+// across it if nothing ended it, which is why this cannot be answered by
+// looking at uses.
+bool AMDGPUPromoteAllocaImpl::isLiveAcrossCall(const AllocaInst *AI) const {
+ if (CallBlocks.empty())
+ return false;
+
+ const Function &F = *AI->getFunction();
+
+ // Without an explicit start the object is live from the alloca, so a call
+ // anywhere after it counts, and the alloca is in the entry block.
+ bool HaveStart = false;
+ for (const User *U : AI->users()) {
+ const auto *II = dyn_cast<IntrinsicInst>(U);
+ if (II && II->getIntrinsicID() == Intrinsic::lifetime_start) {
+ HaveStart = true;
+ break;
+ }
+ }
+
+ // Live-in state per block, to a fixed point over the CFG. Walking a block
+ // from its live-in state gives its live-out, and a call seen while live is
+ // the answer.
+ DenseMap<const BasicBlock *, bool> LiveIn;
+ SmallVector<const BasicBlock *> Worklist;
+
+ // Walk a block, returning whether the object is live on exit, and reporting
+ // whether a call is reached while it is live.
+ const auto scan = [&](const BasicBlock &BB, bool Live, bool *SawCall) {
+ for (const Instruction &I : BB) {
+ if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
+ Intrinsic::ID ID = II->getIntrinsicID();
+ if ((ID == Intrinsic::lifetime_start ||
+ ID == Intrinsic::lifetime_end) &&
+ II->getArgOperand(0)->stripPointerCasts() == AI) {
+ Live = ID == Intrinsic::lifetime_start;
+ continue;
+ }
+ continue;
+ }
+ if (!HaveStart && &I == static_cast<const Instruction *>(AI))
+ Live = true;
+ if (Live && SawCall && isa<CallBase>(&I))
+ *SawCall = true;
+ }
+ return Live;
+ };
+
+ LiveIn[&F.getEntryBlock()] = false;
+ Worklist.push_back(&F.getEntryBlock());
+ while (!Worklist.empty()) {
+ const BasicBlock *BB = Worklist.pop_back_val();
+ bool Out = scan(*BB, LiveIn[BB], nullptr);
+ for (const BasicBlock *Succ : successors(BB)) {
+ auto It = LiveIn.find(Succ);
+ if (It == LiveIn.end()) {
+ LiveIn[Succ] = Out;
+ Worklist.push_back(Succ);
+ } else if (Out && !It->second) {
+ It->second = true;
+ Worklist.push_back(Succ);
+ }
+ }
+ }
+
+ for (const BasicBlock *BB : CallBlocks) {
+ auto It = LiveIn.find(BB);
+ if (It == LiveIn.end())
+ continue; // unreachable
+ bool SawCall = false;
+ scan(*BB, It->second, &SawCall);
+ if (SawCall)
+ return true;
+ }
+ return false;
+}
+
// Decide whether an alloca can be moved into the VGPR ("as memory") address
// space, where it lives in registers rather than in scratch and is reached with
// an indexed register access instead of a load or store.
@@ -1441,6 +1530,14 @@ void AMDGPUPromoteAllocaImpl::analyzePromoteToVGPR(AllocaAnalysis &AA) const {
if (!AMDGPU::isVGPRLoadStoreSupported(Bits, Bits, Alignment))
return Reject(Inst, "unsupported access size or alignment");
+ // Be stricter than the lowering about alignment. It derives the dword
+ // index as pointer >> 2, so an access that is not naturally aligned reads
+ // or writes the dword containing it rather than straddling two. Whole
+ // dword accesses are tolerated there without complaint; an object should
+ // not be moved into a place where that starts happening to it.
+ if (Alignment.value() < std::min<uint64_t>(Bits / 8, 4))
+ return Reject(Inst, "insufficiently aligned access");
+
continue;
}
@@ -1478,6 +1575,11 @@ void AMDGPUPromoteAllocaImpl::analyzePromoteToVGPR(AllocaAnalysis &AA) const {
return Reject(Inst, "unhandled alloca user");
}
+ // Last, because it is the only check here that walks the function rather than
+ // the alloca's own uses.
+ if (isLiveAcrossCall(AA.Alloca))
+ return Reject(AA.Alloca, "live across a call");
+
AA.VGPR.Enable = true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
index e79286a838b29..0efb27a7b255a 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
@@ -6,6 +6,19 @@
; RUN: opt -S -mtriple=amdgpu12.00-- -passes=amdgpu-promote-alloca -o - %s \
; RUN: | FileCheck %s --check-prefix=OFF
+; Promoting decides for itself whether an object can live in registers, and the
+; backend decides the same thing again and reports an error when it cannot. The
+; two have to agree: promoting something the backend then rejects turns a
+; working program into a failed compile. Feeding the promoted IR straight to the
+; backend is what holds them together - these two runs fail, with the backend's
+; own diagnostic, the moment promoting admits something the backend does not.
+; RUN: opt -S -mtriple=amdgpu12.00-- -passes=amdgpu-promote-alloca \
+; RUN: -amdgpu-promote-private -o - %s \
+; RUN: | llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null
+; RUN: opt -S -mtriple=amdgpu12.00-- -passes=amdgpu-promote-alloca \
+; RUN: -amdgpu-promote-private -o - %s \
+; RUN: | llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null
+
; A private alloca can be moved into the VGPR "as memory" address space (13),
; where it lives in registers rather than in scratch. The pointers derived from
; it change address space and the object is then allocated exactly as one
@@ -122,10 +135,9 @@ define amdgpu_kernel void @not_promoted_call(ptr addrspace(1) %out, i32 %i) {
define amdgpu_kernel void @not_promoted_misaligned(ptr addrspace(1) %out, i32 %i) {
; CHECK-LABEL: define amdgpu_kernel void @not_promoted_misaligned(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
-; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
-; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
-; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
-; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 1
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 1
; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
@@ -250,6 +262,173 @@ define amdgpu_kernel void @not_promoted_memcpy(ptr addrspace(1) %out, i32 %i) {
ret void
}
+declare void @llvm.lifetime.start.p5(ptr addrspace(5) nocapture)
+declare void @llvm.lifetime.end.p5(ptr addrspace(5) nocapture)
+
+; A call only matters where the object is live, which the lifetime markers
+; bound. The backend decides this the same way, so promoting here cannot
+; produce something it then rejects.
+define amdgpu_kernel void @call_before_lifetime(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @call_before_lifetime(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @extern()
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @call_before_lifetime(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: call void @extern()
+; OFF-NEXT: call void @llvm.lifetime.start.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @llvm.lifetime.end.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ call void @extern()
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %obj)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %v = load i32, ptr addrspace(5) %p, align 4
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %obj)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @call_after_lifetime(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @call_after_lifetime(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: call void @extern()
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @call_after_lifetime(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: call void @llvm.lifetime.start.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @llvm.lifetime.end.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: call void @extern()
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %obj)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %v = load i32, ptr addrspace(5) %p, align 4
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %obj)
+ call void @extern()
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; The call is on a path the object is not live on.
+define amdgpu_kernel void @call_other_branch(ptr addrspace(1) %out, i32 %i, i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @call_other_branch(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: br i1 [[C]], label %[[SLOW:.*]], label %[[DONE:.*]]
+; CHECK: [[SLOW]]:
+; CHECK-NEXT: call void @extern()
+; CHECK-NEXT: br label %[[DONE]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @call_other_branch(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i1 [[C:%.*]]) {
+; OFF-NEXT: [[ENTRY:.*:]]
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: call void @llvm.lifetime.start.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @llvm.lifetime.end.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: br i1 [[C]], label %[[SLOW:.*]], label %[[DONE:.*]]
+; OFF: [[SLOW]]:
+; OFF-NEXT: call void @extern()
+; OFF-NEXT: br label %[[DONE]]
+; OFF: [[DONE]]:
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+entry:
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %obj)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %v = load i32, ptr addrspace(5) %p, align 4
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %obj)
+ br i1 %c, label %slow, label %done
+slow:
+ call void @extern()
+ br label %done
+done:
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+; Genuinely spanning the live range: still refused.
+define amdgpu_kernel void @call_spans_lifetime(ptr addrspace(1) %out, i32 %i) {
+; CHECK-LABEL: define amdgpu_kernel void @call_spans_lifetime(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: call void @llvm.lifetime.start.p5(ptr addrspace(5) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: call void @extern()
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.end.p5(ptr addrspace(5) [[OBJ]])
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @call_spans_lifetime(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: call void @llvm.lifetime.start.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @extern()
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: call void @llvm.lifetime.end.p5(ptr addrspace(5) [[OBJ]])
+; OFF-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %obj)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ call void @extern()
+ %v = load i32, ptr addrspace(5) %p, align 4
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %obj)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
; An intrinsic overloaded on the pointer type has the address space in its
; mangled name, so moving the object has to rebuild the call, or the name no
; longer describes the argument.
>From 4d2edff4b58850f9acd630a5dc2c713931e0c50e Mon Sep 17 00:00:00 2001
From: Gheorghe-Teodor Bercea <dobercea at amd.com>
Date: Tue, 25 Aug 2026 15:19:12 -0400
Subject: [PATCH 4/4] Count the intrinsics that really become calls when
refusing promotion
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 39 ++++++---
.../AddressSpaceVGPR/as-vgpr-promote.ll | 81 +++++++++++++++++++
2 files changed, 111 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 0a070e20950fa..ffd133440ebfb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -244,6 +244,33 @@ class AMDGPUPromoteAlloca : public FunctionPass {
bool NoOpt;
};
+// Whether \p I becomes a call the object would have to survive.
+//
+// An object in the VGPR address space occupies caller-saved registers and
+// cannot be spilled, so one live across a call has nowhere to be and
+// AMDGPUPrivateObjectVGPRs refuses it. Promotion has to predict that refusal
+// exactly: too lax and a working program becomes a failed compile.
+//
+// Nearly every intrinsic lowers to instructions rather than to a call, and
+// counting those would refuse promotion almost everywhere. The two that really
+// do become one have to count - see the switch in
+// AMDGPUCallLowering::lowerCall and the matching cases in SelectionDAGBuilder.
+static bool isCallForLiveness(const Instruction &I) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ return false;
+ if (const auto *II = dyn_cast<IntrinsicInst>(CB)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::amdgcn_call_whole_wave:
+ case Intrinsic::amdgcn_cs_chain:
+ return true;
+ default:
+ return false;
+ }
+ }
+ return true;
+}
+
static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
const Function &F) {
const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
@@ -430,16 +457,11 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) {
// in analyzePromoteToVGPR. Collected once here because the answer is a
// property of the function, not of the object.
//
- // Intrinsics do not count: on this target they lower to instructions rather
- // than to calls. Were one ever to lower to a call, the result would be the
- // backend's diagnostic rather than a wrong answer.
CallBlocks.clear();
if (PromoteToVGPR) {
- for (const Instruction &I : instructions(F)) {
- const auto *CB = dyn_cast<CallBase>(&I);
- if (CB && !isa<IntrinsicInst>(CB))
+ for (const Instruction &I : instructions(F))
+ if (isCallForLiveness(I))
CallBlocks.insert(I.getParent());
- }
}
bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
@@ -1453,11 +1475,10 @@ bool AMDGPUPromoteAllocaImpl::isLiveAcrossCall(const AllocaInst *AI) const {
Live = ID == Intrinsic::lifetime_start;
continue;
}
- continue;
}
if (!HaveStart && &I == static_cast<const Instruction *>(AI))
Live = true;
- if (Live && SawCall && isa<CallBase>(&I))
+ if (Live && SawCall && isCallForLiveness(I))
*SawCall = true;
}
return Live;
diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
index 0efb27a7b255a..edda56bb595e6 100644
--- a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
+++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-promote.ll
@@ -598,6 +598,87 @@ define amdgpu_kernel void @vector_preferred(ptr addrspace(1) %out, i32 %i) {
store i32 %v, ptr addrspace(1) %out
ret void
}
+; Almost every intrinsic lowers to instructions, so one in the live range must
+; not stand in the way of promotion.
+declare float @llvm.fabs.f32(float)
+
+define amdgpu_kernel void @intrinsic_is_not_a_call(ptr addrspace(1) %out, i32 %i, float %f) {
+; CHECK-LABEL: define amdgpu_kernel void @intrinsic_is_not_a_call(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], float [[F:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0]]
+; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]])
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(13) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[A:%.*]] = call float @llvm.fabs.f32(float [[F]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4
+; CHECK-NEXT: [[B:%.*]] = bitcast float [[A]] to i32
+; CHECK-NEXT: [[S:%.*]] = add i32 [[V]], [[B]]
+; CHECK-NEXT: store i32 [[S]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_kernel void @intrinsic_is_not_a_call(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], float [[F:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[A:%.*]] = call float @llvm.fabs.f32(float [[F]])
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[B:%.*]] = bitcast float [[A]] to i32
+; OFF-NEXT: [[S:%.*]] = add i32 [[V]], [[B]]
+; OFF-NEXT: store i32 [[S]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %a = call float @llvm.fabs.f32(float %f)
+ %v = load i32, ptr addrspace(5) %p, align 4
+ %b = bitcast float %a to i32
+ %s = add i32 %v, %b
+ store i32 %s, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; A whole-wave call is an intrinsic that really does become a call, so an object
+; live across it cannot be promoted. Getting this wrong does not cost an
+; optimization: the object would be promoted and the backend would then refuse
+; it, turning a working program into a failed compile. The llc run above is what
+; would catch that.
+declare amdgpu_gfx_whole_wave i32 @whole_wave_callee(i1 %active, i32 %x, i32 %y, i32 inreg %c)
+
+define amdgpu_cs void @not_promoted_whole_wave_call(ptr addrspace(1) %out, i32 %i, i32 inreg %c) {
+; CHECK-LABEL: define amdgpu_cs void @not_promoted_whole_wave_call(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 inreg [[C:%.*]]) {
+; CHECK-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; CHECK-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: [[R:%.*]] = call i32 (ptr, ...) @llvm.amdgcn.call.whole.wave.i32.p0(ptr @whole_wave_callee, i32 [[I]], i32 [[I]], i32 inreg [[C]])
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: [[S:%.*]] = add i32 [[V]], [[R]]
+; CHECK-NEXT: store i32 [[S]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+; OFF-LABEL: define amdgpu_cs void @not_promoted_whole_wave_call(
+; OFF-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 inreg [[C:%.*]]) {
+; OFF-NEXT: [[OBJ:%.*]] = alloca [8 x i32], align 4, addrspace(5)
+; OFF-NEXT: [[P:%.*]] = getelementptr i8, ptr addrspace(5) [[OBJ]], i32 [[I]]
+; OFF-NEXT: store i32 7, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[R:%.*]] = call i32 (ptr, ...) @llvm.amdgcn.call.whole.wave.i32.p0(ptr @whole_wave_callee, i32 [[I]], i32 [[I]], i32 inreg [[C]])
+; OFF-NEXT: [[V:%.*]] = load i32, ptr addrspace(5) [[P]], align 4
+; OFF-NEXT: [[S:%.*]] = add i32 [[V]], [[R]]
+; OFF-NEXT: store i32 [[S]], ptr addrspace(1) [[OUT]], align 4
+; OFF-NEXT: ret void
+;
+ %obj = alloca [8 x i32], align 4, addrspace(5)
+ %p = getelementptr i8, ptr addrspace(5) %obj, i32 %i
+ store i32 7, ptr addrspace(5) %p, align 4
+ %r = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @whole_wave_callee, i32 %i, i32 %i, i32 inreg %c)
+ %v = load i32, ptr addrspace(5) %p, align 4
+ %s = add i32 %v, %r
+ store i32 %s, ptr addrspace(1) %out, align 4
+ ret void
+}
+
;.
; CHECK: [[META0]] = !{i32 0, i32 32}
;.
More information about the llvm-branch-commits
mailing list