[llvm] [AMDGPU][NewPass] Attempt to promote uniform ptr arguments to inreg (PR #210410)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 06:46:45 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/210410
>From 88efb481bc1916f8de55e04907aeb0197b0aa491 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 17 Jul 2026 14:01:47 -0500
Subject: [PATCH 1/4] try to promote uniform ptr args to SGPRs
---
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 ++++++-
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 +-
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 541 ++++++++++++++++++
4 files changed, 775 insertions(+), 33 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 630ffad96e451..4a9d369076b4d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,8 +14,16 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -30,6 +38,15 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1434,6 +1451,186 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
+ AMDGPUInformationCache &InfoCache) {
+ const DataLayout &DL = M.getDataLayout();
+ TargetMachine &TM = InfoCache.TM;
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ const UniformityInfo *UI =
+ InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
+ *Caller);
+ if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged)
+ InfoCache.invalidateAnalyses();
+ }
+ }
+ return Changed;
+}
+
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1667,7 +1864,10 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- return A.run() == ChangeStatus::CHANGED;
+ bool PromoteChanged =
+ promoteUniformPointerArgsToInReg(M, AG, InfoCache);
+ bool AttChanged = A.run() == ChangeStatus::CHANGED;
+ return AttChanged || PromoteChanged;
}
} // namespace
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index cf4cb5f644bf9..ccb2ed7a54121 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,6 +298,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 5, i32 6}
-; CHECK: [[META1]] = !{}
+; CHECK: [[META0]] = !{i32 2, i32 10}
+; CHECK: [[META1]] = !{i32 5, i32 6}
+; CHECK: [[META2]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index f9edbd070ae7c..c95e6537fd0ab 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
new file mode 100644
index 0000000000000..81e85cd211a40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -0,0 +1,541 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+
+; A uniform pointer argument of an internal function, passed from a kernel,
+; is promoted to inreg (SGPR) on both the definition and the call site.
+
+; CHECK-LABEL: define internal fastcc void @callee_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
+; CHECK: call fastcc void @callee_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; A divergent pointer operand (derived from the workitem id) must NOT be
+; promoted, because inreg would drop all but one lane's value.
+
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_divergent(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
+; CHECK: call fastcc void @callee_divergent(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_divergent(ptr %pdiv)
+ ret void
+}
+
+; Private pointers name lane-private storage. Even if the pointer value itself
+; is uniform, the callee must not learn that the pointee is wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_private(
+; CHECK: call fastcc void @callee_private(ptr addrspace(5) %a)
+ %a = alloca i32, addrspace(5)
+ call fastcc void @callee_private(ptr addrspace(5) %a)
+ ret void
+}
+
+; A flat pointer derived from private memory carries the same risk.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_flat_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_private(
+; CHECK: call fastcc void @callee_flat_private(ptr %f)
+ %a = alloca i32, addrspace(5)
+ %f = addrspacecast ptr addrspace(5) %a to ptr
+ call fastcc void @callee_flat_private(ptr %f)
+ ret void
+}
+
+; A flat pointer that may be private on one path of a phi/select must not be
+; promoted, even though the other path is a benign global pointer.
+
+; CHECK-LABEL: define internal fastcc void @callee_phi_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_phi_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+ at gvar = addrspace(1) global i32 0
+
+define amdgpu_kernel void @k_phi_private(i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
+; CHECK: call fastcc void @callee_phi_private(ptr %sel)
+ %a = alloca i32, addrspace(5)
+ %fa = addrspacecast ptr addrspace(5) %a to ptr
+ %fg = addrspacecast ptr addrspace(1) @gvar to ptr
+ %sel = select i1 %c, ptr %fa, ptr %fg
+ call fastcc void @callee_phi_private(ptr %sel)
+ ret void
+}
+
+; A private pointer reached through a long getelementptr chain stays in the
+; private address space, so the address-space check still blocks it.
+
+; CHECK-LABEL: define internal fastcc void @callee_deep_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_deep_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_deep_private(
+; CHECK: call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ %a = alloca [64 x i32], addrspace(5)
+ %g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
+ %g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
+ %g3 = getelementptr i32, ptr addrspace(5) %g2, i32 1
+ %g4 = getelementptr i32, ptr addrspace(5) %g3, i32 1
+ %g5 = getelementptr i32, ptr addrspace(5) %g4, i32 1
+ %g6 = getelementptr i32, ptr addrspace(5) %g5, i32 1
+ %g7 = getelementptr i32, ptr addrspace(5) %g6, i32 1
+ %g8 = getelementptr i32, ptr addrspace(5) %g7, i32 1
+ %g9 = getelementptr i32, ptr addrspace(5) %g8, i32 1
+ %g10 = getelementptr i32, ptr addrspace(5) %g9, i32 1
+ %g11 = getelementptr i32, ptr addrspace(5) %g10, i32 1
+ %g12 = getelementptr i32, ptr addrspace(5) %g11, i32 1
+ call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ ret void
+}
+
+; Callee-side guard: even a uniform flat pointer must not be promoted if the
+; callee reinterprets it as private (scratch), because that extracts a
+; lane-relative offset that is not wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_casts_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_casts_private(ptr %p) {
+ %q = addrspacecast ptr %p to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_casts_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
+; CHECK: call fastcc void @callee_casts_private(ptr %p)
+ call fastcc void @callee_casts_private(ptr %p)
+ ret void
+}
+
+; Positive control: a flat load/store through the argument is the common, safe
+; use and must still be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_load(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_flat_load(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_load(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
+; CHECK: call fastcc void @callee_flat_load(ptr inreg %p)
+ call fastcc void @callee_flat_load(ptr %p)
+ ret void
+}
+
+; SGPR dword budget (default 8): only the first four pointer arguments fit;
+; the fifth is left in VGPRs.
+
+; CHECK-LABEL: define internal fastcc void @callee_budget(
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+ ret void
+}
+
+define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+; CHECK-LABEL: define amdgpu_kernel void @k_budget(
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+ call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
+ ret void
+}
+
+; TTI cross-check: a flat load is NeverUniform in GCNTTI even when the address
+; is wave-uniform, so the operand must not be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_tti_flatload(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at gptr = addrspace(1) global ptr null
+
+define amdgpu_kernel void @k_tti_flatload() {
+; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload(
+; CHECK: call fastcc void @callee_tti_flatload(ptr %p)
+ %flatg = addrspacecast ptr addrspace(1) @gptr to ptr
+ %p = load ptr, ptr %flatg
+ call fastcc void @callee_tti_flatload(ptr %p)
+ ret void
+}
+
+; External linkage: all call sites are not necessarily visible, so the ABI
+; must not be changed.
+
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define fastcc void @callee_external(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_external(ptr %p) {
+ call fastcc void @callee_external(ptr %p)
+ ret void
+}
+
+; Address-taken internal function: an indirect call we cannot see could pass a
+; divergent value, so do not promote.
+
+; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_addrtaken(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+ at fnptr = global ptr null
+
+define amdgpu_kernel void @k_addrtaken(ptr %p) {
+ store ptr @callee_addrtaken, ptr @fnptr
+ call fastcc void @callee_addrtaken(ptr %p)
+ ret void
+}
+
+; Arguments with ABI-affecting attributes must not be promoted.
+
+%struct.Foo = type { i32 }
+
+; CHECK-LABEL: define internal fastcc void @callee_byref(
+; CHECK-SAME: ptr byref(%struct.Foo) %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_byref(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_byref(
+; CHECK: call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc ptr @callee_returned(
+; CHECK-SAME: ptr returned %p
+; CHECK-NOT: ptr inreg
+define internal fastcc ptr @callee_returned(ptr returned %p) {
+ ret ptr %p
+}
+
+define amdgpu_kernel void @k_returned(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_returned(
+; CHECK: call fastcc ptr @callee_returned(ptr returned %p)
+ %r = call fastcc ptr @callee_returned(ptr returned %p)
+ store ptr %r, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
+; CHECK-SAME: ptr swiftasync %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_swiftasync(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
+; CHECK: call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_hidden(
+; CHECK-SAME: ptr "amdgpu-hidden-argument" %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_hidden(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
+; CHECK: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ ret void
+}
+
+; Non-pointer uniform arguments are out of scope for this pass.
+
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 %n
+; CHECK-NOT: i32 inreg
+define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+ store i32 %n, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; The pointer is still promoted, the scalar is not.
+; CHECK: call fastcc void @callee_scalar(i32 %n, ptr inreg %p)
+ call fastcc void @callee_scalar(i32 %n, ptr %p)
+ ret void
+}
+
+; Mixed call sites: one uniform, one divergent. A single divergent operand must
+; block promotion, since the definition is shared by all callers.
+
+; CHECK-LABEL: define internal fastcc void @callee_mixed(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_mixed(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
+; CHECK: call fastcc void @callee_mixed(ptr %p)
+ call fastcc void @callee_mixed(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
+; CHECK: call fastcc void @callee_mixed(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_mixed(ptr %pdiv)
+ ret void
+}
+
+; Multi-hop chain: uniformity propagates from the kernel through each internal
+; function to a fixpoint, so every hop's closure pointer is promoted.
+
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_leaf(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_mid(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_mid(ptr %p) {
+; CHECK: call fastcc void @chain_leaf(ptr inreg %p)
+ call fastcc void @chain_leaf(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_top(ptr %p) {
+; CHECK: call fastcc void @chain_mid(ptr inreg %p)
+ call fastcc void @chain_mid(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_chain(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_chain(
+; CHECK: call fastcc void @chain_top(ptr inreg %p)
+ call fastcc void @chain_top(ptr %p)
+ ret void
+}
+
+; Fixpoint must converge regardless of the order the functions appear in the
+; module (callees defined before/after their callers) and across a diamond.
+
+; CHECK-LABEL: define internal fastcc void @scram_a(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_a(ptr %p) {
+ call fastcc void @scram_b(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_c(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_c(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_b(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_b(ptr %p) {
+ call fastcc void @scram_c(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_scram(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scram(
+; CHECK: call fastcc void @scram_a(ptr inreg %p)
+ call fastcc void @scram_a(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_bot(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_bot(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_l(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_l(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_r(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_r(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_top(ptr %p) {
+ call fastcc void @diam_l(ptr %p)
+ call fastcc void @diam_r(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_diam(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_diam(
+; CHECK: call fastcc void @diam_top(ptr inreg %p)
+ call fastcc void @diam_top(ptr %p)
+ ret void
+}
+
+; An argument that is already inreg must be left untouched (no double attribute,
+; no crash).
+
+; CHECK-LABEL: define internal fastcc void @callee_already(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_already(ptr inreg %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_already(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_already(
+; CHECK: call fastcc void @callee_already(ptr inreg %p)
+ call fastcc void @callee_already(ptr inreg %p)
+ ret void
+}
+
+; Invoke call sites are not audited for inreg ABI consistency under exceptional
+; control flow, so promotion is skipped.
+
+; CHECK-LABEL: define internal fastcc void @callee_invoke(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_invoke(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
+; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
+; CHECK: invoke fastcc void @callee_invoke(ptr %p)
+ invoke fastcc void @callee_invoke(ptr %p) to label %cont unwind label %lpad
+
+cont:
+ ret void
+
+lpad:
+ %tok = landingpad { ptr, i32 }
+ cleanup
+ ret void
+}
+
+; Indirect call through a bitcast of the function pointer: the callee is not a
+; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
+
+; CHECK-LABEL: define internal fastcc void @callee_bitcast(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_bitcast(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_bitcast(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
+; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+ %fn = bitcast ptr @callee_bitcast to ptr
+ call void %fn(ptr %p)
+ ret void
+}
+
+; A function stored into a global (non-call use) is not eligible even if there
+; is also a direct call the pass can see.
+
+; CHECK-LABEL: define internal fastcc void @callee_stored(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_stored(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at fn_slot = global ptr null
+
+define amdgpu_kernel void @k_stored(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_stored(
+; CHECK: call fastcc void @callee_stored(ptr %p)
+ store ptr @callee_stored, ptr @fn_slot
+ call fastcc void @callee_stored(ptr %p)
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
>From 45c2507ee5ef8ec6ac8e61e9bffc4a2cb8e32102 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 13:55:38 -0500
Subject: [PATCH 2/4] create new pass to promote uniform ptr args
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 5 +
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 +-------------
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 251 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 ++--
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 45 +++-
9 files changed, 336 insertions(+), 236 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c72fa69aa1419..f77206cf6b63f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -314,6 +314,11 @@ struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
bool GlobalOpt;
};
+struct AMDGPUPromoteUniformArgsPass
+ : PassInfoMixin<AMDGPUPromoteUniformArgsPass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &);
extern char &AMDGPULowerExecSyncLegacyPassID;
ModulePass *createAMDGPULowerExecSyncLegacyPass();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 4a9d369076b4d..630ffad96e451 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,16 +14,8 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -38,15 +30,6 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
-STATISTIC(NumPromotedInRegArgs,
- "Number of uniform pointer arguments promoted to inreg");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
-
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1451,186 +1434,6 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
-}
-
-static bool isEligibleInRegUniformCallee(const Function &F) {
- if (F.isDeclaration() || F.isVarArg())
- return false;
- if (!F.hasLocalLinkage() || F.hasAddressTaken())
- return false;
- switch (F.getCallingConv()) {
- case CallingConv::C:
- case CallingConv::Fast:
- break;
- default:
- return false;
- }
- for (const User *U : F.users()) {
- const auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall() || isa<InvokeInst>(CB))
- return false;
- }
- if (F.user_empty())
- return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
- return true;
-}
-
-static bool mayBePrivateDerivedPointer(const Value *V) {
- assert(V->getType()->isPointerTy());
- if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
-
- SmallVector<const Value *, 8> Objects;
- getUnderlyingObjects(V, Objects);
- for (const Value *Obj : Objects) {
- if (isa<AllocaInst>(Obj))
- return true;
- if (Obj->getType()->isPointerTy() &&
- Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- }
- return false;
-}
-
-static bool calleeCastsArgToPrivate(const Argument &A) {
- SmallVector<const Value *, 16> Worklist;
- SmallPtrSet<const Value *, 16> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
- if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(ASC);
- continue;
- }
- if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
- if (II->getType()->getPointerAddressSpace() ==
- AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(II);
- }
- continue;
- }
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
- Worklist.push_back(U);
- }
- }
- return false;
-}
-
-static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall())
- return false;
- if (isa<InvokeInst>(CB))
- return false;
- Calls.push_back(CB);
- }
- return !Calls.empty();
-}
-
-static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
- AMDGPUInformationCache &InfoCache) {
- const DataLayout &DL = M.getDataLayout();
- TargetMachine &TM = InfoCache.TM;
- bool Changed = false;
- bool RoundChanged = true;
- while (RoundChanged) {
- RoundChanged = false;
- for (Function &F : M) {
- if (!isEligibleInRegUniformCallee(F))
- continue;
-
- SmallVector<CallBase *, 8> Calls;
- if (!collectCallSites(F, Calls))
- continue;
-
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
- bool FuncChanged = false;
- for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
- continue;
- if (calleeCastsArgToPrivate(A))
- continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
- continue;
- }
-
- bool AllUniform = true;
- for (CallBase *CB : Calls) {
- Value *ArgOp = CB->getArgOperand(A.getArgNo());
- if (mayBePrivateDerivedPointer(ArgOp)) {
- AllUniform = false;
- break;
- }
-
- Function *Caller = CB->getFunction();
- const UniformityInfo *UI =
- InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
- *Caller);
- if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
- AllUniform = false;
- break;
- }
-
- TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
- if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
- AllUniform = false;
- break;
- }
- }
- if (!AllUniform)
- continue;
-
- A.addAttr(Attribute::InReg);
- for (CallBase *CB : Calls)
- CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
- ++NumPromotedInRegArgs;
- FuncChanged = Changed = RoundChanged = true;
- }
-
- if (FuncChanged)
- InfoCache.invalidateAnalyses();
- }
- }
- return Changed;
-}
-
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1864,10 +1667,7 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- bool PromoteChanged =
- promoteUniformPointerArgsToInReg(M, AG, InfoCache);
- bool AttChanged = A.run() == ChangeStatus::CHANGED;
- return AttChanged || PromoteChanged;
+ return A.run() == ChangeStatus::CHANGED;
}
} // namespace
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d052f3c73920c..6a90ff2522870 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -17,6 +17,7 @@
#define MODULE_PASS(NAME, CREATE_PASS)
#endif
MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
+MODULE_PASS("amdgpu-promote-uniform-args", AMDGPUPromoteUniformArgsPass())
MODULE_PASS("amdgpu-export-kernel-runtime-handles", AMDGPUExportKernelRuntimeHandlesPass())
MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
AMDGPULowerBufferFatPointersPass(*this))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
new file mode 100644
index 0000000000000..dd689d4667ad5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -0,0 +1,251 @@
+//===-- AMDGPUPromoteUniformArgs.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// For non-entry AMDGPU functions, pointer arguments are passed in VGPRs unless
+// marked \c inreg. When such an argument is actually uniform across the
+// wavefront at every visible call site, passing it in VGPRs forces every lane
+// to carry the same value and can inflate register pressure. This pass promotes
+// provably-uniform pointer arguments of internal callees to \c inreg (SGPR
+// passing) on the definition and at each direct call site.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-promote-uniform-args"
+
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumPromotedInRegFuncs,
+ "Number of functions with a promoted uniform pointer argument");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<bool> EnablePromoteUniformPointerArgs(
+ "amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
+ cl::desc("Promote provably uniform internal pointer arguments to inreg"));
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
+namespace {
+
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M,
+ ModuleAnalysisManager &AM) {
+ auto &FAM =
+ AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ const DataLayout &DL = M.getDataLayout();
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ UniformityInfo &UI =
+ FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ const TargetTransformInfo &TTI =
+ FAM.getResult<TargetIRAnalysis>(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged) {
+ ++NumPromotedInRegFuncs;
+ FAM.invalidate(F, PreservedAnalyses::none());
+ SmallPtrSet<Function *, 8> InvalidatedCallers;
+ for (CallBase *CB : Calls) {
+ Function *Caller = CB->getFunction();
+ if (Caller != &F && InvalidatedCallers.insert(Caller).second)
+ FAM.invalidate(*Caller, PreservedAnalyses::none());
+ }
+ }
+ }
+ }
+ return Changed;
+}
+
+} // namespace
+
+PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ return PreservedAnalyses::all();
+ return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
+ : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8c2732630e2e6..ce2aaacfb6969 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1101,6 +1101,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
ThinOrFullLTOPhase Phase) {
if (Level != OptimizationLevel::O0) {
if (!isLTOPreLink(Phase)) {
+ if (Level > OptimizationLevel::O1 && getTargetTriple().isAMDGCN())
+ MPM.addPass(AMDGPUPromoteUniformArgsPass());
if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
AMDGPUAttributorOptions Opts;
MPM.addPass(AMDGPUAttributorPass(*this, Opts, Phase));
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..40afd303343f9 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -102,6 +102,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUPrintfRuntimeBinding.cpp
AMDGPUPromoteAlloca.cpp
AMDGPUPromoteKernelArguments.cpp
+ AMDGPUPromoteUniformArgs.cpp
AMDGPURegBankCombiner.cpp
AMDGPURegBankLegalize.cpp
AMDGPURegBankLegalizeHelper.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index ccb2ed7a54121..cf4cb5f644bf9 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,7 +298,6 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 2, i32 10}
-; CHECK: [[META1]] = !{i32 5, i32 6}
-; CHECK: [[META2]] = !{}
+; CHECK: [[META0]] = !{i32 5, i32 6}
+; CHECK: [[META1]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index c95e6537fd0ab..f9edbd070ae7c 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 81e85cd211a40..1568fbb3e561f 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -511,7 +511,8 @@ define internal fastcc void @callee_bitcast(ptr %p) {
define amdgpu_kernel void @k_bitcast(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+; CHECK: call void {{.*}}(ptr {{.*}}%p)
+; CHECK-NOT: inreg
%fn = bitcast ptr @callee_bitcast to ptr
call void %fn(ptr %p)
ret void
@@ -538,4 +539,44 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; inlinehint and alwaysinline callees behave the same for inreg promotion when
+; the callee remains an out-of-line call (the motivating Kokkos case).
+
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
+; CHECK: call fastcc void @callee_inlinehint_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
+; CHECK: call fastcc void @callee_alwaysinline_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+attributes #0 = { inlinehint }
+attributes #1 = { alwaysinline }
+
declare i32 @llvm.amdgcn.workitem.id.x()
>From 97276e5e9b6c98e63b8d32bfedfd49f0d827c1bb Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 14:23:53 -0500
Subject: [PATCH 3/4] code format
---
.../Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index dd689d4667ad5..9bf13cb6b58e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -38,8 +38,9 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
+STATISTIC(
+ NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
@@ -158,8 +159,7 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
- auto &FAM =
- AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
@@ -200,8 +200,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
}
Function *Caller = CB->getFunction();
- UniformityInfo &UI =
- FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(*Caller);
if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
AllUniform = false;
break;
@@ -243,8 +242,9 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
} // namespace
PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
- ModuleAnalysisManager &AM) {
- if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs ||
+ !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
: PreservedAnalyses::all();
>From 5ede768d47f41ac6b3b9842a17f8d859ba6c8dac Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 31 Jul 2026 08:46:18 -0500
Subject: [PATCH 4/4] update hard coded list, musttail scan, preserve CFG
analysis, SGPR budget drop
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 87 ++++++++++---------
.../CodeGen/AMDGPU/promote-uniform-args.ll | 30 +++++--
2 files changed, 69 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 9bf13cb6b58e3..d29881c8e7965 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -23,12 +23,12 @@
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Analysis.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
using namespace llvm;
@@ -38,32 +38,21 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(
- NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
cl::desc("Promote provably uniform internal pointer arguments to inreg"));
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
namespace {
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
+static bool canPromoteArgToInReg(const Argument &A) {
+ if (!A.getType()->isPointerTy() || A.hasInRegAttr())
+ return false;
+ if (A.hasPointeeInMemoryValueAttr() || A.hasNestAttr() ||
+ A.hasReturnedAttr() || A.hasSwiftSelfAttr() || A.hasSwiftErrorAttr() ||
+ A.hasAttribute(Attribute::SwiftAsync))
+ return false;
+ return !A.hasAttribute("amdgpu-hidden-argument");
}
static bool isEligibleInRegUniformCallee(const Function &F) {
@@ -87,14 +76,31 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
}
if (F.user_empty())
return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
return true;
}
+static bool argForwardedByMustTail(const Argument &A) {
+ SmallVector<const Value *, 8> Worklist;
+ SmallPtrSet<const Value *, 8> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *CB = dyn_cast<CallBase>(U)) {
+ if (CB->isMustTailCall())
+ return true;
+ continue;
+ }
+ if (isa<BitCastInst, GetElementPtrInst, AddrSpaceCastInst, PHINode,
+ SelectInst>(U))
+ Worklist.push_back(cast<Instruction>(U));
+ }
+ }
+ return false;
+}
+
static bool mayBePrivateDerivedPointer(const Value *V) {
assert(V->getType()->isPointerTy());
if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
@@ -160,7 +166,6 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
while (RoundChanged) {
@@ -173,23 +178,14 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
if (!collectCallSites(F, Calls))
continue;
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
bool FuncChanged = false;
for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ if (!canPromoteArgToInReg(A))
continue;
- if (calleeCastsArgToPrivate(A))
+ if (argForwardedByMustTail(A))
continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
+ if (calleeCastsArgToPrivate(A))
continue;
- }
bool AllUniform = true;
for (CallBase *CB : Calls) {
@@ -219,19 +215,21 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
A.addAttr(Attribute::InReg);
for (CallBase *CB : Calls)
CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
++NumPromotedInRegArgs;
FuncChanged = Changed = RoundChanged = true;
}
if (FuncChanged) {
++NumPromotedInRegFuncs;
- FAM.invalidate(F, PreservedAnalyses::none());
+ PreservedAnalyses FuncPA;
+ // Attribute-only change; the CFG is unchanged.
+ FuncPA.preserveSet<CFGAnalyses>();
+ FAM.invalidate(F, FuncPA);
SmallPtrSet<Function *, 8> InvalidatedCallers;
for (CallBase *CB : Calls) {
Function *Caller = CB->getFunction();
if (Caller != &F && InvalidatedCallers.insert(Caller).second)
- FAM.invalidate(*Caller, PreservedAnalyses::none());
+ FAM.invalidate(*Caller, FuncPA);
}
}
}
@@ -246,6 +244,9 @@ PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
if (!EnablePromoteUniformPointerArgs ||
!Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
- return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
- : PreservedAnalyses::all();
+ if (!promoteUniformPointerArgsToInReg(M, AM))
+ return PreservedAnalyses::all();
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 1568fbb3e561f..97b0476278369 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -172,11 +172,10 @@ define amdgpu_kernel void @k_flat_load(ptr %p) {
ret void
}
-; SGPR dword budget (default 8): only the first four pointer arguments fit;
-; the fifth is left in VGPRs.
+; Multiple uniform pointer arguments are all promoted opportunistically.
; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
ret void
@@ -185,7 +184,7 @@ define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4)
call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
ret void
}
@@ -539,6 +538,27 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; musttail forwarding of an argument blocks inreg promotion for that argument.
+
+define internal fastcc void @tail_target(ptr %q) {
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_musttail_forward(ptr %p) {
+ musttail call fastcc void @tail_target(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_musttail_forward(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
+; CHECK: call fastcc void @callee_musttail_forward(ptr %p)
+ call fastcc void @callee_musttail_forward(ptr %p)
+ ret void
+}
+
; inlinehint and alwaysinline callees behave the same for inreg promotion when
; the callee remains an out-of-line call (the motivating Kokkos case).
More information about the llvm-commits
mailing list