[llvm] [AMDGPU][NewPass] Attempt to promote uniform ptr arguments to inreg (PR #210410)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 10:01:09 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/210410
>From 88efb481bc1916f8de55e04907aeb0197b0aa491 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 17 Jul 2026 14:01:47 -0500
Subject: [PATCH 1/6] try to promote uniform ptr args to SGPRs
---
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 ++++++-
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 +-
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 541 ++++++++++++++++++
4 files changed, 775 insertions(+), 33 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 630ffad96e451..4a9d369076b4d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,8 +14,16 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -30,6 +38,15 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1434,6 +1451,186 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
+ AMDGPUInformationCache &InfoCache) {
+ const DataLayout &DL = M.getDataLayout();
+ TargetMachine &TM = InfoCache.TM;
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ const UniformityInfo *UI =
+ InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
+ *Caller);
+ if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged)
+ InfoCache.invalidateAnalyses();
+ }
+ }
+ return Changed;
+}
+
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1667,7 +1864,10 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- return A.run() == ChangeStatus::CHANGED;
+ bool PromoteChanged =
+ promoteUniformPointerArgsToInReg(M, AG, InfoCache);
+ bool AttChanged = A.run() == ChangeStatus::CHANGED;
+ return AttChanged || PromoteChanged;
}
} // namespace
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index cf4cb5f644bf9..ccb2ed7a54121 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,6 +298,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 5, i32 6}
-; CHECK: [[META1]] = !{}
+; CHECK: [[META0]] = !{i32 2, i32 10}
+; CHECK: [[META1]] = !{i32 5, i32 6}
+; CHECK: [[META2]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index f9edbd070ae7c..c95e6537fd0ab 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
new file mode 100644
index 0000000000000..81e85cd211a40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -0,0 +1,541 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+
+; A uniform pointer argument of an internal function, passed from a kernel,
+; is promoted to inreg (SGPR) on both the definition and the call site.
+
+; CHECK-LABEL: define internal fastcc void @callee_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
+; CHECK: call fastcc void @callee_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; A divergent pointer operand (derived from the workitem id) must NOT be
+; promoted, because inreg would drop all but one lane's value.
+
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_divergent(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
+; CHECK: call fastcc void @callee_divergent(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_divergent(ptr %pdiv)
+ ret void
+}
+
+; Private pointers name lane-private storage. Even if the pointer value itself
+; is uniform, the callee must not learn that the pointee is wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_private(
+; CHECK: call fastcc void @callee_private(ptr addrspace(5) %a)
+ %a = alloca i32, addrspace(5)
+ call fastcc void @callee_private(ptr addrspace(5) %a)
+ ret void
+}
+
+; A flat pointer derived from private memory carries the same risk.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_flat_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_private(
+; CHECK: call fastcc void @callee_flat_private(ptr %f)
+ %a = alloca i32, addrspace(5)
+ %f = addrspacecast ptr addrspace(5) %a to ptr
+ call fastcc void @callee_flat_private(ptr %f)
+ ret void
+}
+
+; A flat pointer that may be private on one path of a phi/select must not be
+; promoted, even though the other path is a benign global pointer.
+
+; CHECK-LABEL: define internal fastcc void @callee_phi_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_phi_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+ at gvar = addrspace(1) global i32 0
+
+define amdgpu_kernel void @k_phi_private(i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
+; CHECK: call fastcc void @callee_phi_private(ptr %sel)
+ %a = alloca i32, addrspace(5)
+ %fa = addrspacecast ptr addrspace(5) %a to ptr
+ %fg = addrspacecast ptr addrspace(1) @gvar to ptr
+ %sel = select i1 %c, ptr %fa, ptr %fg
+ call fastcc void @callee_phi_private(ptr %sel)
+ ret void
+}
+
+; A private pointer reached through a long getelementptr chain stays in the
+; private address space, so the address-space check still blocks it.
+
+; CHECK-LABEL: define internal fastcc void @callee_deep_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_deep_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_deep_private(
+; CHECK: call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ %a = alloca [64 x i32], addrspace(5)
+ %g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
+ %g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
+ %g3 = getelementptr i32, ptr addrspace(5) %g2, i32 1
+ %g4 = getelementptr i32, ptr addrspace(5) %g3, i32 1
+ %g5 = getelementptr i32, ptr addrspace(5) %g4, i32 1
+ %g6 = getelementptr i32, ptr addrspace(5) %g5, i32 1
+ %g7 = getelementptr i32, ptr addrspace(5) %g6, i32 1
+ %g8 = getelementptr i32, ptr addrspace(5) %g7, i32 1
+ %g9 = getelementptr i32, ptr addrspace(5) %g8, i32 1
+ %g10 = getelementptr i32, ptr addrspace(5) %g9, i32 1
+ %g11 = getelementptr i32, ptr addrspace(5) %g10, i32 1
+ %g12 = getelementptr i32, ptr addrspace(5) %g11, i32 1
+ call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ ret void
+}
+
+; Callee-side guard: even a uniform flat pointer must not be promoted if the
+; callee reinterprets it as private (scratch), because that extracts a
+; lane-relative offset that is not wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_casts_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_casts_private(ptr %p) {
+ %q = addrspacecast ptr %p to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_casts_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
+; CHECK: call fastcc void @callee_casts_private(ptr %p)
+ call fastcc void @callee_casts_private(ptr %p)
+ ret void
+}
+
+; Positive control: a flat load/store through the argument is the common, safe
+; use and must still be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_load(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_flat_load(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_load(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
+; CHECK: call fastcc void @callee_flat_load(ptr inreg %p)
+ call fastcc void @callee_flat_load(ptr %p)
+ ret void
+}
+
+; SGPR dword budget (default 8): only the first four pointer arguments fit;
+; the fifth is left in VGPRs.
+
+; CHECK-LABEL: define internal fastcc void @callee_budget(
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+ ret void
+}
+
+define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+; CHECK-LABEL: define amdgpu_kernel void @k_budget(
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+ call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
+ ret void
+}
+
+; TTI cross-check: a flat load is NeverUniform in GCNTTI even when the address
+; is wave-uniform, so the operand must not be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_tti_flatload(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at gptr = addrspace(1) global ptr null
+
+define amdgpu_kernel void @k_tti_flatload() {
+; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload(
+; CHECK: call fastcc void @callee_tti_flatload(ptr %p)
+ %flatg = addrspacecast ptr addrspace(1) @gptr to ptr
+ %p = load ptr, ptr %flatg
+ call fastcc void @callee_tti_flatload(ptr %p)
+ ret void
+}
+
+; External linkage: all call sites are not necessarily visible, so the ABI
+; must not be changed.
+
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define fastcc void @callee_external(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_external(ptr %p) {
+ call fastcc void @callee_external(ptr %p)
+ ret void
+}
+
+; Address-taken internal function: an indirect call we cannot see could pass a
+; divergent value, so do not promote.
+
+; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_addrtaken(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+ at fnptr = global ptr null
+
+define amdgpu_kernel void @k_addrtaken(ptr %p) {
+ store ptr @callee_addrtaken, ptr @fnptr
+ call fastcc void @callee_addrtaken(ptr %p)
+ ret void
+}
+
+; Arguments with ABI-affecting attributes must not be promoted.
+
+%struct.Foo = type { i32 }
+
+; CHECK-LABEL: define internal fastcc void @callee_byref(
+; CHECK-SAME: ptr byref(%struct.Foo) %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_byref(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_byref(
+; CHECK: call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc ptr @callee_returned(
+; CHECK-SAME: ptr returned %p
+; CHECK-NOT: ptr inreg
+define internal fastcc ptr @callee_returned(ptr returned %p) {
+ ret ptr %p
+}
+
+define amdgpu_kernel void @k_returned(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_returned(
+; CHECK: call fastcc ptr @callee_returned(ptr returned %p)
+ %r = call fastcc ptr @callee_returned(ptr returned %p)
+ store ptr %r, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
+; CHECK-SAME: ptr swiftasync %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_swiftasync(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
+; CHECK: call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_hidden(
+; CHECK-SAME: ptr "amdgpu-hidden-argument" %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_hidden(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
+; CHECK: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ ret void
+}
+
+; Non-pointer uniform arguments are out of scope for this pass.
+
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 %n
+; CHECK-NOT: i32 inreg
+define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+ store i32 %n, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; The pointer is still promoted, the scalar is not.
+; CHECK: call fastcc void @callee_scalar(i32 %n, ptr inreg %p)
+ call fastcc void @callee_scalar(i32 %n, ptr %p)
+ ret void
+}
+
+; Mixed call sites: one uniform, one divergent. A single divergent operand must
+; block promotion, since the definition is shared by all callers.
+
+; CHECK-LABEL: define internal fastcc void @callee_mixed(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_mixed(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
+; CHECK: call fastcc void @callee_mixed(ptr %p)
+ call fastcc void @callee_mixed(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
+; CHECK: call fastcc void @callee_mixed(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_mixed(ptr %pdiv)
+ ret void
+}
+
+; Multi-hop chain: uniformity propagates from the kernel through each internal
+; function to a fixpoint, so every hop's closure pointer is promoted.
+
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_leaf(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_mid(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_mid(ptr %p) {
+; CHECK: call fastcc void @chain_leaf(ptr inreg %p)
+ call fastcc void @chain_leaf(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_top(ptr %p) {
+; CHECK: call fastcc void @chain_mid(ptr inreg %p)
+ call fastcc void @chain_mid(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_chain(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_chain(
+; CHECK: call fastcc void @chain_top(ptr inreg %p)
+ call fastcc void @chain_top(ptr %p)
+ ret void
+}
+
+; Fixpoint must converge regardless of the order the functions appear in the
+; module (callees defined before/after their callers) and across a diamond.
+
+; CHECK-LABEL: define internal fastcc void @scram_a(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_a(ptr %p) {
+ call fastcc void @scram_b(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_c(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_c(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_b(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_b(ptr %p) {
+ call fastcc void @scram_c(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_scram(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scram(
+; CHECK: call fastcc void @scram_a(ptr inreg %p)
+ call fastcc void @scram_a(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_bot(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_bot(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_l(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_l(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_r(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_r(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_top(ptr %p) {
+ call fastcc void @diam_l(ptr %p)
+ call fastcc void @diam_r(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_diam(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_diam(
+; CHECK: call fastcc void @diam_top(ptr inreg %p)
+ call fastcc void @diam_top(ptr %p)
+ ret void
+}
+
+; An argument that is already inreg must be left untouched (no double attribute,
+; no crash).
+
+; CHECK-LABEL: define internal fastcc void @callee_already(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_already(ptr inreg %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_already(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_already(
+; CHECK: call fastcc void @callee_already(ptr inreg %p)
+ call fastcc void @callee_already(ptr inreg %p)
+ ret void
+}
+
+; Invoke call sites are not audited for inreg ABI consistency under exceptional
+; control flow, so promotion is skipped.
+
+; CHECK-LABEL: define internal fastcc void @callee_invoke(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_invoke(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
+; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
+; CHECK: invoke fastcc void @callee_invoke(ptr %p)
+ invoke fastcc void @callee_invoke(ptr %p) to label %cont unwind label %lpad
+
+cont:
+ ret void
+
+lpad:
+ %tok = landingpad { ptr, i32 }
+ cleanup
+ ret void
+}
+
+; Indirect call through a bitcast of the function pointer: the callee is not a
+; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
+
+; CHECK-LABEL: define internal fastcc void @callee_bitcast(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_bitcast(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_bitcast(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
+; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+ %fn = bitcast ptr @callee_bitcast to ptr
+ call void %fn(ptr %p)
+ ret void
+}
+
+; A function stored into a global (non-call use) is not eligible even if there
+; is also a direct call the pass can see.
+
+; CHECK-LABEL: define internal fastcc void @callee_stored(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_stored(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at fn_slot = global ptr null
+
+define amdgpu_kernel void @k_stored(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_stored(
+; CHECK: call fastcc void @callee_stored(ptr %p)
+ store ptr @callee_stored, ptr @fn_slot
+ call fastcc void @callee_stored(ptr %p)
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
>From 45c2507ee5ef8ec6ac8e61e9bffc4a2cb8e32102 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 13:55:38 -0500
Subject: [PATCH 2/6] create new pass to promote uniform ptr args
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 5 +
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 +-------------
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 251 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 ++--
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 45 +++-
9 files changed, 336 insertions(+), 236 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c72fa69aa1419..f77206cf6b63f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -314,6 +314,11 @@ struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
bool GlobalOpt;
};
+struct AMDGPUPromoteUniformArgsPass
+ : PassInfoMixin<AMDGPUPromoteUniformArgsPass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &);
extern char &AMDGPULowerExecSyncLegacyPassID;
ModulePass *createAMDGPULowerExecSyncLegacyPass();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 4a9d369076b4d..630ffad96e451 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,16 +14,8 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -38,15 +30,6 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
-STATISTIC(NumPromotedInRegArgs,
- "Number of uniform pointer arguments promoted to inreg");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
-
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1451,186 +1434,6 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
-}
-
-static bool isEligibleInRegUniformCallee(const Function &F) {
- if (F.isDeclaration() || F.isVarArg())
- return false;
- if (!F.hasLocalLinkage() || F.hasAddressTaken())
- return false;
- switch (F.getCallingConv()) {
- case CallingConv::C:
- case CallingConv::Fast:
- break;
- default:
- return false;
- }
- for (const User *U : F.users()) {
- const auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall() || isa<InvokeInst>(CB))
- return false;
- }
- if (F.user_empty())
- return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
- return true;
-}
-
-static bool mayBePrivateDerivedPointer(const Value *V) {
- assert(V->getType()->isPointerTy());
- if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
-
- SmallVector<const Value *, 8> Objects;
- getUnderlyingObjects(V, Objects);
- for (const Value *Obj : Objects) {
- if (isa<AllocaInst>(Obj))
- return true;
- if (Obj->getType()->isPointerTy() &&
- Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- }
- return false;
-}
-
-static bool calleeCastsArgToPrivate(const Argument &A) {
- SmallVector<const Value *, 16> Worklist;
- SmallPtrSet<const Value *, 16> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
- if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(ASC);
- continue;
- }
- if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
- if (II->getType()->getPointerAddressSpace() ==
- AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(II);
- }
- continue;
- }
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
- Worklist.push_back(U);
- }
- }
- return false;
-}
-
-static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall())
- return false;
- if (isa<InvokeInst>(CB))
- return false;
- Calls.push_back(CB);
- }
- return !Calls.empty();
-}
-
-static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
- AMDGPUInformationCache &InfoCache) {
- const DataLayout &DL = M.getDataLayout();
- TargetMachine &TM = InfoCache.TM;
- bool Changed = false;
- bool RoundChanged = true;
- while (RoundChanged) {
- RoundChanged = false;
- for (Function &F : M) {
- if (!isEligibleInRegUniformCallee(F))
- continue;
-
- SmallVector<CallBase *, 8> Calls;
- if (!collectCallSites(F, Calls))
- continue;
-
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
- bool FuncChanged = false;
- for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
- continue;
- if (calleeCastsArgToPrivate(A))
- continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
- continue;
- }
-
- bool AllUniform = true;
- for (CallBase *CB : Calls) {
- Value *ArgOp = CB->getArgOperand(A.getArgNo());
- if (mayBePrivateDerivedPointer(ArgOp)) {
- AllUniform = false;
- break;
- }
-
- Function *Caller = CB->getFunction();
- const UniformityInfo *UI =
- InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
- *Caller);
- if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
- AllUniform = false;
- break;
- }
-
- TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
- if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
- AllUniform = false;
- break;
- }
- }
- if (!AllUniform)
- continue;
-
- A.addAttr(Attribute::InReg);
- for (CallBase *CB : Calls)
- CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
- ++NumPromotedInRegArgs;
- FuncChanged = Changed = RoundChanged = true;
- }
-
- if (FuncChanged)
- InfoCache.invalidateAnalyses();
- }
- }
- return Changed;
-}
-
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1864,10 +1667,7 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- bool PromoteChanged =
- promoteUniformPointerArgsToInReg(M, AG, InfoCache);
- bool AttChanged = A.run() == ChangeStatus::CHANGED;
- return AttChanged || PromoteChanged;
+ return A.run() == ChangeStatus::CHANGED;
}
} // namespace
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d052f3c73920c..6a90ff2522870 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -17,6 +17,7 @@
#define MODULE_PASS(NAME, CREATE_PASS)
#endif
MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
+MODULE_PASS("amdgpu-promote-uniform-args", AMDGPUPromoteUniformArgsPass())
MODULE_PASS("amdgpu-export-kernel-runtime-handles", AMDGPUExportKernelRuntimeHandlesPass())
MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
AMDGPULowerBufferFatPointersPass(*this))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
new file mode 100644
index 0000000000000..dd689d4667ad5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -0,0 +1,251 @@
+//===-- AMDGPUPromoteUniformArgs.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// For non-entry AMDGPU functions, pointer arguments are passed in VGPRs unless
+// marked \c inreg. When such an argument is actually uniform across the
+// wavefront at every visible call site, passing it in VGPRs forces every lane
+// to carry the same value and can inflate register pressure. This pass promotes
+// provably-uniform pointer arguments of internal callees to \c inreg (SGPR
+// passing) on the definition and at each direct call site.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-promote-uniform-args"
+
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumPromotedInRegFuncs,
+ "Number of functions with a promoted uniform pointer argument");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<bool> EnablePromoteUniformPointerArgs(
+ "amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
+ cl::desc("Promote provably uniform internal pointer arguments to inreg"));
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
+namespace {
+
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M,
+ ModuleAnalysisManager &AM) {
+ auto &FAM =
+ AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ const DataLayout &DL = M.getDataLayout();
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ UniformityInfo &UI =
+ FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ const TargetTransformInfo &TTI =
+ FAM.getResult<TargetIRAnalysis>(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged) {
+ ++NumPromotedInRegFuncs;
+ FAM.invalidate(F, PreservedAnalyses::none());
+ SmallPtrSet<Function *, 8> InvalidatedCallers;
+ for (CallBase *CB : Calls) {
+ Function *Caller = CB->getFunction();
+ if (Caller != &F && InvalidatedCallers.insert(Caller).second)
+ FAM.invalidate(*Caller, PreservedAnalyses::none());
+ }
+ }
+ }
+ }
+ return Changed;
+}
+
+} // namespace
+
+PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ return PreservedAnalyses::all();
+ return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
+ : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8c2732630e2e6..ce2aaacfb6969 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1101,6 +1101,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
ThinOrFullLTOPhase Phase) {
if (Level != OptimizationLevel::O0) {
if (!isLTOPreLink(Phase)) {
+ if (Level > OptimizationLevel::O1 && getTargetTriple().isAMDGCN())
+ MPM.addPass(AMDGPUPromoteUniformArgsPass());
if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
AMDGPUAttributorOptions Opts;
MPM.addPass(AMDGPUAttributorPass(*this, Opts, Phase));
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..40afd303343f9 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -102,6 +102,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUPrintfRuntimeBinding.cpp
AMDGPUPromoteAlloca.cpp
AMDGPUPromoteKernelArguments.cpp
+ AMDGPUPromoteUniformArgs.cpp
AMDGPURegBankCombiner.cpp
AMDGPURegBankLegalize.cpp
AMDGPURegBankLegalizeHelper.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index ccb2ed7a54121..cf4cb5f644bf9 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,7 +298,6 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 2, i32 10}
-; CHECK: [[META1]] = !{i32 5, i32 6}
-; CHECK: [[META2]] = !{}
+; CHECK: [[META0]] = !{i32 5, i32 6}
+; CHECK: [[META1]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index c95e6537fd0ab..f9edbd070ae7c 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 81e85cd211a40..1568fbb3e561f 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -511,7 +511,8 @@ define internal fastcc void @callee_bitcast(ptr %p) {
define amdgpu_kernel void @k_bitcast(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+; CHECK: call void {{.*}}(ptr {{.*}}%p)
+; CHECK-NOT: inreg
%fn = bitcast ptr @callee_bitcast to ptr
call void %fn(ptr %p)
ret void
@@ -538,4 +539,44 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; inlinehint and alwaysinline callees behave the same for inreg promotion when
+; the callee remains an out-of-line call (the motivating Kokkos case).
+
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
+; CHECK: call fastcc void @callee_inlinehint_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
+; CHECK: call fastcc void @callee_alwaysinline_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+attributes #0 = { inlinehint }
+attributes #1 = { alwaysinline }
+
declare i32 @llvm.amdgcn.workitem.id.x()
>From 97276e5e9b6c98e63b8d32bfedfd49f0d827c1bb Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 14:23:53 -0500
Subject: [PATCH 3/6] code format
---
.../Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index dd689d4667ad5..9bf13cb6b58e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -38,8 +38,9 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
+STATISTIC(
+ NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
@@ -158,8 +159,7 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
- auto &FAM =
- AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
@@ -200,8 +200,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
}
Function *Caller = CB->getFunction();
- UniformityInfo &UI =
- FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(*Caller);
if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
AllUniform = false;
break;
@@ -243,8 +242,9 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
} // namespace
PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
- ModuleAnalysisManager &AM) {
- if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs ||
+ !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
: PreservedAnalyses::all();
>From 5ede768d47f41ac6b3b9842a17f8d859ba6c8dac Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 31 Jul 2026 08:46:18 -0500
Subject: [PATCH 4/6] update hard coded list, musttail scan, preserve CFG
analysis, SGPR budget drop
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 87 ++++++++++---------
.../CodeGen/AMDGPU/promote-uniform-args.ll | 30 +++++--
2 files changed, 69 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 9bf13cb6b58e3..d29881c8e7965 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -23,12 +23,12 @@
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Analysis.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
using namespace llvm;
@@ -38,32 +38,21 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(
- NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
cl::desc("Promote provably uniform internal pointer arguments to inreg"));
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
namespace {
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
+static bool canPromoteArgToInReg(const Argument &A) {
+ if (!A.getType()->isPointerTy() || A.hasInRegAttr())
+ return false;
+ if (A.hasPointeeInMemoryValueAttr() || A.hasNestAttr() ||
+ A.hasReturnedAttr() || A.hasSwiftSelfAttr() || A.hasSwiftErrorAttr() ||
+ A.hasAttribute(Attribute::SwiftAsync))
+ return false;
+ return !A.hasAttribute("amdgpu-hidden-argument");
}
static bool isEligibleInRegUniformCallee(const Function &F) {
@@ -87,14 +76,31 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
}
if (F.user_empty())
return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
return true;
}
+static bool argForwardedByMustTail(const Argument &A) {
+ SmallVector<const Value *, 8> Worklist;
+ SmallPtrSet<const Value *, 8> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *CB = dyn_cast<CallBase>(U)) {
+ if (CB->isMustTailCall())
+ return true;
+ continue;
+ }
+ if (isa<BitCastInst, GetElementPtrInst, AddrSpaceCastInst, PHINode,
+ SelectInst>(U))
+ Worklist.push_back(cast<Instruction>(U));
+ }
+ }
+ return false;
+}
+
static bool mayBePrivateDerivedPointer(const Value *V) {
assert(V->getType()->isPointerTy());
if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
@@ -160,7 +166,6 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
while (RoundChanged) {
@@ -173,23 +178,14 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
if (!collectCallSites(F, Calls))
continue;
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
bool FuncChanged = false;
for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ if (!canPromoteArgToInReg(A))
continue;
- if (calleeCastsArgToPrivate(A))
+ if (argForwardedByMustTail(A))
continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
+ if (calleeCastsArgToPrivate(A))
continue;
- }
bool AllUniform = true;
for (CallBase *CB : Calls) {
@@ -219,19 +215,21 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
A.addAttr(Attribute::InReg);
for (CallBase *CB : Calls)
CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
++NumPromotedInRegArgs;
FuncChanged = Changed = RoundChanged = true;
}
if (FuncChanged) {
++NumPromotedInRegFuncs;
- FAM.invalidate(F, PreservedAnalyses::none());
+ PreservedAnalyses FuncPA;
+ // Attribute-only change; the CFG is unchanged.
+ FuncPA.preserveSet<CFGAnalyses>();
+ FAM.invalidate(F, FuncPA);
SmallPtrSet<Function *, 8> InvalidatedCallers;
for (CallBase *CB : Calls) {
Function *Caller = CB->getFunction();
if (Caller != &F && InvalidatedCallers.insert(Caller).second)
- FAM.invalidate(*Caller, PreservedAnalyses::none());
+ FAM.invalidate(*Caller, FuncPA);
}
}
}
@@ -246,6 +244,9 @@ PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
if (!EnablePromoteUniformPointerArgs ||
!Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
- return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
- : PreservedAnalyses::all();
+ if (!promoteUniformPointerArgsToInReg(M, AM))
+ return PreservedAnalyses::all();
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 1568fbb3e561f..97b0476278369 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -172,11 +172,10 @@ define amdgpu_kernel void @k_flat_load(ptr %p) {
ret void
}
-; SGPR dword budget (default 8): only the first four pointer arguments fit;
-; the fifth is left in VGPRs.
+; Multiple uniform pointer arguments are all promoted opportunistically.
; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
ret void
@@ -185,7 +184,7 @@ define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4)
call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
ret void
}
@@ -539,6 +538,27 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; musttail forwarding of an argument blocks inreg promotion for that argument.
+
+define internal fastcc void @tail_target(ptr %q) {
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_musttail_forward(ptr %p) {
+ musttail call fastcc void @tail_target(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_musttail_forward(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
+; CHECK: call fastcc void @callee_musttail_forward(ptr %p)
+ call fastcc void @callee_musttail_forward(ptr %p)
+ ret void
+}
+
; inlinehint and alwaysinline callees behave the same for inreg promotion when
; the callee remains an out-of-line call (the motivating Kokkos case).
>From eed465b8167c2dbc4653775b452d4f76fc91cb5d Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 31 Jul 2026 11:33:04 -0500
Subject: [PATCH 5/6] code format
---
llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index d29881c8e7965..2d0e5b9060b78 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -22,8 +22,8 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Attributes.h"
#include "llvm/IR/Analysis.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
>From 4c3eb3888d99529b4b750a837787c1ea6a8c4360 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 14 Aug 2026 11:47:25 -0500
Subject: [PATCH 6/6] Replace UniformityAnalysis with a conservative trivial
uniformity check (TTI AlwaysUniform plus recursive propagation through
visible caller chains), and add a path-based cycle guard for recursive arg
forwarding.
Fix two correctness bugs before promotion:
- Skip any function containing a musttail call; adding inreg to any parameter breaks positional ABI matching, not just forwarded args.
- Follow freeze, ptrmask, and pointer-aliasing intrinsics when detecting private address-space casts in the callee.
additional tests for various promotion accept/reject use cases, tests for the musttail, freeze, and ptrmask cases.
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 156 ++--
.../CodeGen/AMDGPU/promote-uniform-args.ll | 882 +++++++++++++++---
2 files changed, 855 insertions(+), 183 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 2d0e5b9060b78..59d77698aae30 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -6,12 +6,12 @@
//
//===----------------------------------------------------------------------===//
//
-// For non-entry AMDGPU functions, pointer arguments are passed in VGPRs unless
-// marked \c inreg. When such an argument is actually uniform across the
-// wavefront at every visible call site, passing it in VGPRs forces every lane
-// to carry the same value and can inflate register pressure. This pass promotes
-// provably-uniform pointer arguments of internal callees to \c inreg (SGPR
-// passing) on the definition and at each direct call site.
+// Promote provably-uniform pointer arguments of internal callees to \c inreg
+// (SGPR passing) on the definition and at every direct call site. Uniformity is
+// established conservatively: an operand is uniform if it is an always-uniform
+// instruction (per TargetTransformInfo) or an argument that every visible
+// caller forwards from another uniform value. A full \c UniformityInfo-based
+// check is a planned follow-up.
//
//===----------------------------------------------------------------------===//
@@ -20,7 +20,6 @@
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Attributes.h"
@@ -39,8 +38,8 @@ STATISTIC(NumPromotedInRegArgs,
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-static cl::opt<bool> EnablePromoteUniformPointerArgs(
- "amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
+static cl::opt<bool> EnablePromoteUniformArgs(
+ "amdgpu-enable-promote-uniform-args", cl::Hidden, cl::init(true),
cl::desc("Promote provably uniform internal pointer arguments to inreg"));
namespace {
@@ -79,25 +78,14 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
return true;
}
-static bool argForwardedByMustTail(const Argument &A) {
- SmallVector<const Value *, 8> Worklist;
- SmallPtrSet<const Value *, 8> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *CB = dyn_cast<CallBase>(U)) {
+// musttail requires the enclosing function's parameter ABI (including inreg) to
+// match positionally, so skip promotion for any function containing one.
+static bool hasMustTailCall(const Function &F) {
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
if (CB->isMustTailCall())
return true;
- continue;
- }
- if (isa<BitCastInst, GetElementPtrInst, AddrSpaceCastInst, PHINode,
- SelectInst>(U))
- Worklist.push_back(cast<Instruction>(U));
- }
- }
return false;
}
@@ -134,38 +122,97 @@ static bool calleeCastsArgToPrivate(const Argument &A) {
continue;
}
if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::amdgcn_addrspacecast_nonnull:
if (II->getType()->getPointerAddressSpace() ==
AMDGPUAS::PRIVATE_ADDRESS)
return true;
Worklist.push_back(II);
+ break;
+ case Intrinsic::ptrmask:
+ case Intrinsic::launder_invariant_group:
+ case Intrinsic::strip_invariant_group:
+ Worklist.push_back(II);
+ break;
+ default:
+ break;
}
continue;
}
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst, FreezeInst>(
+ U))
Worklist.push_back(U);
}
}
return false;
}
-static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall())
- return false;
- if (isa<InvokeInst>(CB))
- return false;
- Calls.push_back(CB);
+static bool isTriviallyUniformArg(
+ const Argument *Arg,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited);
+
+static bool isTriviallyUniformOperand(
+ const Value *V,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited) {
+ if (const auto *Arg = dyn_cast<Argument>(V))
+ return isTriviallyUniformArg(Arg, GetTTI, Visited);
+
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ const TargetTransformInfo &TTI = GetTTI(*I->getFunction());
+ return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
+ }
+
+ return false;
+}
+
+static bool isTriviallyUniformArg(
+ const Argument *Arg,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited) {
+ if (Arg->hasInRegAttr())
+ return true;
+
+ const Function *F = Arg->getParent();
+ if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+ return AMDGPU::isArgPassedInSGPR(Arg);
+
+ // Only direct-call-only internal callees have fully visible callers; an
+ // indirect or address-taken caller could pass a divergent value we never see.
+ if (!isEligibleInRegUniformCallee(*F))
+ return false;
+
+ // Path-based cycle guard: a (mutually) recursive chain that forwards the
+ // argument back to itself cannot be proven here and must terminate. The
+ // argument is removed on the way out so a diamond-shaped call graph can still
+ // reach the same argument through an independent path.
+ if (!Visited.insert(Arg).second)
+ return false;
+
+ bool HasCallSite = false;
+ bool AllUniform = true;
+ for (const User *U : F->users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != F)
+ continue;
+ HasCallSite = true;
+ if (!isTriviallyUniformOperand(CB->getArgOperand(Arg->getArgNo()), GetTTI,
+ Visited)) {
+ AllUniform = false;
+ break;
+ }
}
- return !Calls.empty();
+ Visited.erase(Arg);
+ return HasCallSite && AllUniform;
}
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto GetTTI = [&FAM](const Function &F) -> TargetTransformInfo & {
+ return FAM.getResult<TargetIRAnalysis>(const_cast<Function &>(F));
+ };
bool Changed = false;
bool RoundChanged = true;
while (RoundChanged) {
@@ -174,16 +221,26 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
if (!isEligibleInRegUniformCallee(F))
continue;
+ // Promoting any argument changes F's ABI attributes, which a musttail
+ // call in F's body would then no longer match positionally. Skip
+ // conservatively.
+ if (hasMustTailCall(F))
+ continue;
+
+ // isEligibleInRegUniformCallee guarantees every user is a direct,
+ // non-musttail, non-invoke call to F, so we can just gather them.
SmallVector<CallBase *, 8> Calls;
- if (!collectCallSites(F, Calls))
+ for (User *U : F.users())
+ if (auto *CB = dyn_cast<CallBase>(U);
+ CB && CB->getCalledFunction() == &F)
+ Calls.push_back(CB);
+ if (Calls.empty())
continue;
bool FuncChanged = false;
for (Argument &A : F.args()) {
if (!canPromoteArgToInReg(A))
continue;
- if (argForwardedByMustTail(A))
- continue;
if (calleeCastsArgToPrivate(A))
continue;
@@ -195,16 +252,8 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
break;
}
- Function *Caller = CB->getFunction();
- UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(*Caller);
- if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
- AllUniform = false;
- break;
- }
-
- const TargetTransformInfo &TTI =
- FAM.getResult<TargetIRAnalysis>(*Caller);
- if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ SmallPtrSet<const Argument *, 4> VisitedArgs;
+ if (!isTriviallyUniformOperand(ArgOp, GetTTI, VisitedArgs)) {
AllUniform = false;
break;
}
@@ -241,8 +290,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
ModuleAnalysisManager &AM) {
- if (!EnablePromoteUniformPointerArgs ||
- !Triple(M.getTargetTriple()).isAMDGCN())
+ if (!EnablePromoteUniformArgs || !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
if (!promoteUniformPointerArgsToInReg(M, AM))
return PreservedAnalyses::all();
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 97b0476278369..64bff2fc40644 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,11 +1,17 @@
-; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -verify-each -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
-; CHECK-LABEL: define internal fastcc void @callee_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+; CHECK-LABEL: define internal fastcc void @callee_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -14,7 +20,11 @@ define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
define amdgpu_kernel void @k_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
-; CHECK: call fastcc void @callee_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_uniform(ptr %p, i32 %tid)
ret void
@@ -23,10 +33,13 @@ define amdgpu_kernel void @k_uniform(ptr %p) {
; A divergent pointer operand (derived from the workitem id) must NOT be
; promoted, because inreg would drop all but one lane's value.
-; CHECK-LABEL: define internal fastcc void @callee_divergent(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_divergent(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -34,7 +47,12 @@ define internal fastcc void @callee_divergent(ptr %p) {
define amdgpu_kernel void @k_divergent(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
-; CHECK: call fastcc void @callee_divergent(ptr %pdiv)
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @callee_divergent(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%pdiv = getelementptr float, ptr %base, i32 %tid
call fastcc void @callee_divergent(ptr %pdiv)
@@ -44,17 +62,22 @@ define amdgpu_kernel void @k_divergent(ptr %base) {
; Private pointers name lane-private storage. Even if the pointer value itself
; is uniform, the callee must not learn that the pointee is wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_private(
-; CHECK-SAME: ptr addrspace(5) %p
-; CHECK-NOT: ptr addrspace(5) inreg
define internal fastcc void @callee_private(ptr addrspace(5) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_private(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr addrspace(5) %p
ret void
}
define amdgpu_kernel void @k_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_private(
-; CHECK: call fastcc void @callee_private(ptr addrspace(5) %a)
+; CHECK-LABEL: define amdgpu_kernel void @k_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: call fastcc void @callee_private(ptr addrspace(5) [[A]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
call fastcc void @callee_private(ptr addrspace(5) %a)
ret void
@@ -62,17 +85,23 @@ define amdgpu_kernel void @k_private() {
; A flat pointer derived from private memory carries the same risk.
-; CHECK-LABEL: define internal fastcc void @callee_flat_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_flat_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_flat_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr %p
ret void
}
define amdgpu_kernel void @k_flat_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_flat_private(
-; CHECK: call fastcc void @callee_flat_private(ptr %f)
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: [[F:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
+; CHECK-NEXT: call fastcc void @callee_flat_private(ptr [[F]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
%f = addrspacecast ptr addrspace(5) %a to ptr
call fastcc void @callee_flat_private(ptr %f)
@@ -82,10 +111,12 @@ define amdgpu_kernel void @k_flat_private() {
; A flat pointer that may be private on one path of a phi/select must not be
; promoted, even though the other path is a benign global pointer.
-; CHECK-LABEL: define internal fastcc void @callee_phi_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_phi_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_phi_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr %p
ret void
}
@@ -94,7 +125,14 @@ define internal fastcc void @callee_phi_private(ptr %p) {
define amdgpu_kernel void @k_phi_private(i1 %c) {
; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
-; CHECK: call fastcc void @callee_phi_private(ptr %sel)
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: [[FA:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
+; CHECK-NEXT: [[FG:%.*]] = addrspacecast ptr addrspace(1) @gvar to ptr
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr [[FA]], ptr [[FG]]
+; CHECK-NEXT: call fastcc void @callee_phi_private(ptr [[SEL]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
%fa = addrspacecast ptr addrspace(5) %a to ptr
%fg = addrspacecast ptr addrspace(1) @gvar to ptr
@@ -106,17 +144,34 @@ define amdgpu_kernel void @k_phi_private(i1 %c) {
; A private pointer reached through a long getelementptr chain stays in the
; private address space, so the address-space check still blocks it.
-; CHECK-LABEL: define internal fastcc void @callee_deep_private(
-; CHECK-SAME: ptr addrspace(5) %p
-; CHECK-NOT: ptr addrspace(5) inreg
define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_deep_private(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr addrspace(5) %p
ret void
}
define amdgpu_kernel void @k_deep_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_deep_private(
-; CHECK: call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+; CHECK-LABEL: define amdgpu_kernel void @k_deep_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca [64 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[G1:%.*]] = getelementptr i32, ptr addrspace(5) [[A]], i32 1
+; CHECK-NEXT: [[G2:%.*]] = getelementptr i32, ptr addrspace(5) [[G1]], i32 1
+; CHECK-NEXT: [[G3:%.*]] = getelementptr i32, ptr addrspace(5) [[G2]], i32 1
+; CHECK-NEXT: [[G4:%.*]] = getelementptr i32, ptr addrspace(5) [[G3]], i32 1
+; CHECK-NEXT: [[G5:%.*]] = getelementptr i32, ptr addrspace(5) [[G4]], i32 1
+; CHECK-NEXT: [[G6:%.*]] = getelementptr i32, ptr addrspace(5) [[G5]], i32 1
+; CHECK-NEXT: [[G7:%.*]] = getelementptr i32, ptr addrspace(5) [[G6]], i32 1
+; CHECK-NEXT: [[G8:%.*]] = getelementptr i32, ptr addrspace(5) [[G7]], i32 1
+; CHECK-NEXT: [[G9:%.*]] = getelementptr i32, ptr addrspace(5) [[G8]], i32 1
+; CHECK-NEXT: [[G10:%.*]] = getelementptr i32, ptr addrspace(5) [[G9]], i32 1
+; CHECK-NEXT: [[G11:%.*]] = getelementptr i32, ptr addrspace(5) [[G10]], i32 1
+; CHECK-NEXT: [[G12:%.*]] = getelementptr i32, ptr addrspace(5) [[G11]], i32 1
+; CHECK-NEXT: call fastcc void @callee_deep_private(ptr addrspace(5) [[G12]])
+; CHECK-NEXT: ret void
+;
%a = alloca [64 x i32], addrspace(5)
%g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
%g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
@@ -138,10 +193,13 @@ define amdgpu_kernel void @k_deep_private() {
; callee reinterprets it as private (scratch), because that extracts a
; lane-relative offset that is not wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_casts_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_casts_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_casts_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
%q = addrspacecast ptr %p to ptr addrspace(5)
store i32 1, ptr addrspace(5) %q
ret void
@@ -149,7 +207,10 @@ define internal fastcc void @callee_casts_private(ptr %p) {
define amdgpu_kernel void @k_casts_private(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
-; CHECK: call fastcc void @callee_casts_private(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_casts_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_casts_private(ptr %p)
ret void
}
@@ -157,9 +218,13 @@ define amdgpu_kernel void @k_casts_private(ptr %p) {
; Positive control: a flat load/store through the argument is the common, safe
; use and must still be promoted.
-; CHECK-LABEL: define internal fastcc void @callee_flat_load(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @callee_flat_load(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_flat_load(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -167,35 +232,45 @@ define internal fastcc void @callee_flat_load(ptr %p) {
define amdgpu_kernel void @k_flat_load(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
-; CHECK: call fastcc void @callee_flat_load(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_flat_load(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_flat_load(ptr %p)
ret void
}
; Multiple uniform pointer arguments are all promoted opportunistically.
-; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
- ptr %p4) {
+; CHECK-LABEL: define internal fastcc void @callee_budget(
+; CHECK-SAME: ptr inreg [[P0:%.*]], ptr inreg [[P1:%.*]], ptr inreg [[P2:%.*]], ptr inreg [[P3:%.*]], ptr inreg [[P4:%.*]]) {
+; CHECK-NEXT: ret void
+;
+ ptr %p4) {
ret void
}
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
- ptr %p4) {
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4)
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_budget(ptr inreg [[P0]], ptr inreg [[P1]], ptr inreg [[P2]], ptr inreg [[P3]], ptr inreg [[P4]])
+; CHECK-NEXT: ret void
+;
+ ptr %p4) {
call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
ret void
}
-; TTI cross-check: a flat load is NeverUniform in GCNTTI even when the address
-; is wave-uniform, so the operand must not be promoted.
+; Trivial check: a loaded pointer is not AlwaysUniform, so the operand must not
+; be promoted even when the load address is wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_tti_flatload(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
@@ -203,8 +278,12 @@ define internal fastcc void @callee_tti_flatload(ptr %p) {
@gptr = addrspace(1) global ptr null
define amdgpu_kernel void @k_tti_flatload() {
-; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload(
-; CHECK: call fastcc void @callee_tti_flatload(ptr %p)
+; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload() {
+; CHECK-NEXT: [[FLATG:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT: [[P:%.*]] = load ptr, ptr [[FLATG]], align 8
+; CHECK-NEXT: call fastcc void @callee_tti_flatload(ptr [[P]])
+; CHECK-NEXT: ret void
+;
%flatg = addrspacecast ptr addrspace(1) @gptr to ptr
%p = load ptr, ptr %flatg
call fastcc void @callee_tti_flatload(ptr %p)
@@ -214,16 +293,24 @@ define amdgpu_kernel void @k_tti_flatload() {
; External linkage: all call sites are not necessarily visible, so the ABI
; must not be changed.
-; CHECK-LABEL: define fastcc void @callee_external(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define fastcc void @callee_external(ptr %p) {
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
}
define amdgpu_kernel void @k_external(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_external(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_external(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_external(ptr %p)
ret void
}
@@ -231,10 +318,13 @@ define amdgpu_kernel void @k_external(ptr %p) {
; Address-taken internal function: an indirect call we cannot see could pass a
; divergent value, so do not promote.
-; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_addrtaken(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -243,6 +333,12 @@ define internal fastcc void @callee_addrtaken(ptr %p) {
@fnptr = global ptr null
define amdgpu_kernel void @k_addrtaken(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_addrtaken(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @callee_addrtaken, ptr @fnptr, align 8
+; CHECK-NEXT: call fastcc void @callee_addrtaken(ptr [[P]])
+; CHECK-NEXT: ret void
+;
store ptr @callee_addrtaken, ptr @fnptr
call fastcc void @callee_addrtaken(ptr %p)
ret void
@@ -252,77 +348,99 @@ define amdgpu_kernel void @k_addrtaken(ptr %p) {
%struct.Foo = type { i32 }
-; CHECK-LABEL: define internal fastcc void @callee_byref(
-; CHECK-SAME: ptr byref(%struct.Foo) %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_byref(
+; CHECK-SAME: ptr byref([[STRUCT_FOO:%.*]]) [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_byref(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_byref(
-; CHECK: call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_byref(ptr byref([[STRUCT_FOO:%.*]]) [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
ret void
}
-; CHECK-LABEL: define internal fastcc ptr @callee_returned(
-; CHECK-SAME: ptr returned %p
-; CHECK-NOT: ptr inreg
define internal fastcc ptr @callee_returned(ptr returned %p) {
+; CHECK-LABEL: define internal fastcc ptr @callee_returned(
+; CHECK-SAME: ptr returned [[P:%.*]]) {
+; CHECK-NEXT: ret ptr [[P]]
+;
ret ptr %p
}
define amdgpu_kernel void @k_returned(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_returned(
-; CHECK: call fastcc ptr @callee_returned(ptr returned %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call fastcc ptr @callee_returned(ptr returned [[P]])
+; CHECK-NEXT: store ptr [[R]], ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%r = call fastcc ptr @callee_returned(ptr returned %p)
store ptr %r, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
-; CHECK-SAME: ptr swiftasync %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
+; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
+; CHECK-SAME: ptr swiftasync [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_swiftasync(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
-; CHECK: call fastcc void @callee_swiftasync(ptr swiftasync %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_swiftasync(ptr swiftasync [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_swiftasync(ptr swiftasync %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_hidden(
-; CHECK-SAME: ptr "amdgpu-hidden-argument" %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
+; CHECK-LABEL: define internal fastcc void @callee_hidden(
+; CHECK-SAME: ptr "amdgpu-hidden-argument" [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_hidden(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
-; CHECK: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
ret void
}
; Non-pointer uniform arguments are out of scope for this pass.
-; CHECK-LABEL: define internal fastcc void @callee_scalar(
-; CHECK-SAME: i32 %n
-; CHECK-NOT: i32 inreg
define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 [[N:%.*]], ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 %n, ptr %p
ret void
}
define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_scalar(i32 [[N]], ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
; The pointer is still promoted, the scalar is not.
-; CHECK: call fastcc void @callee_scalar(i32 %n, ptr inreg %p)
call fastcc void @callee_scalar(i32 %n, ptr %p)
ret void
}
@@ -330,10 +448,13 @@ define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
; Mixed call sites: one uniform, one divergent. A single divergent operand must
; block promotion, since the definition is shared by all callers.
-; CHECK-LABEL: define internal fastcc void @callee_mixed(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_mixed(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_mixed(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -341,14 +462,22 @@ define internal fastcc void @callee_mixed(ptr %p) {
define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
-; CHECK: call fastcc void @callee_mixed(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_mixed(ptr %p)
ret void
}
define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
-; CHECK: call fastcc void @callee_mixed(ptr %pdiv)
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%pdiv = getelementptr float, ptr %base, i32 %tid
call fastcc void @callee_mixed(ptr %pdiv)
@@ -358,32 +487,42 @@ define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
; Multi-hop chain: uniformity propagates from the kernel through each internal
; function to a fixpoint, so every hop's closure pointer is promoted.
-; CHECK-LABEL: define internal fastcc void @chain_leaf(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @chain_mid(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_mid(ptr %p) {
-; CHECK: call fastcc void @chain_leaf(ptr inreg %p)
+; CHECK-LABEL: define internal fastcc void @chain_mid(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_leaf(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_leaf(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @chain_top(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_top(ptr %p) {
-; CHECK: call fastcc void @chain_mid(ptr inreg %p)
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_mid(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_mid(ptr %p)
ret void
}
define amdgpu_kernel void @k_chain(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_chain(
-; CHECK: call fastcc void @chain_top(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_top(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_top(ptr %p)
ret void
}
@@ -391,58 +530,83 @@ define amdgpu_kernel void @k_chain(ptr %p) {
; Fixpoint must converge regardless of the order the functions appear in the
; module (callees defined before/after their callers) and across a diamond.
-; CHECK-LABEL: define internal fastcc void @scram_a(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_a(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_a(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_b(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_b(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @scram_c(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_c(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_c(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @scram_b(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_b(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_b(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_c(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_c(ptr %p)
ret void
}
define amdgpu_kernel void @k_scram(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scram(
-; CHECK: call fastcc void @scram_a(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_a(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_a(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_bot(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_bot(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_bot(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_l(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_l(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_l(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_bot(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_r(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_r(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_r(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_bot(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_top(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_top(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_top(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_l(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @diam_r(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_l(ptr %p)
call fastcc void @diam_r(ptr %p)
ret void
@@ -450,7 +614,10 @@ define internal fastcc void @diam_top(ptr %p) {
define amdgpu_kernel void @k_diam(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_diam(
-; CHECK: call fastcc void @diam_top(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_top(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_top(ptr %p)
ret void
}
@@ -458,16 +625,22 @@ define amdgpu_kernel void @k_diam(ptr %p) {
; An argument that is already inreg must be left untouched (no double attribute,
; no crash).
-; CHECK-LABEL: define internal fastcc void @callee_already(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @callee_already(ptr inreg %p) {
+; CHECK-LABEL: define internal fastcc void @callee_already(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_already(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_already(
-; CHECK: call fastcc void @callee_already(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_already(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_already(ptr inreg %p)
ret void
}
@@ -475,17 +648,28 @@ define amdgpu_kernel void @k_already(ptr %p) {
; Invoke call sites are not audited for inreg ABI consistency under exceptional
; control flow, so promotion is skipped.
-; CHECK-LABEL: define internal fastcc void @callee_invoke(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_invoke(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_invoke(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
-; CHECK: invoke fastcc void @callee_invoke(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) personality ptr null {
+; CHECK-NEXT: invoke fastcc void @callee_invoke(ptr [[P]])
+; CHECK-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD:.*]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[LPAD]]:
+; CHECK-NEXT: [[TOK:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: ret void
+;
invoke fastcc void @callee_invoke(ptr %p) to label %cont unwind label %lpad
cont:
@@ -493,25 +677,30 @@ cont:
lpad:
%tok = landingpad { ptr, i32 }
- cleanup
+ cleanup
ret void
}
; Indirect call through a bitcast of the function pointer: the callee is not a
; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
-; CHECK-LABEL: define internal fastcc void @callee_bitcast(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_bitcast(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_bitcast(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_bitcast(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK: call void {{.*}}(ptr {{.*}}%p)
-; CHECK-NOT: inreg
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[FN:%.*]] = bitcast ptr @callee_bitcast to ptr
+; CHECK-NEXT: call void [[FN]](ptr [[P]])
+; CHECK-NEXT: ret void
+;
%fn = bitcast ptr @callee_bitcast to ptr
call void %fn(ptr %p)
ret void
@@ -520,10 +709,12 @@ define amdgpu_kernel void @k_bitcast(ptr %p) {
; A function stored into a global (non-call use) is not eligible even if there
; is also a direct call the pass can see.
-; CHECK-LABEL: define internal fastcc void @callee_stored(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_stored(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_stored(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
@@ -532,7 +723,11 @@ define internal fastcc void @callee_stored(ptr %p) {
define amdgpu_kernel void @k_stored(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_stored(
-; CHECK: call fastcc void @callee_stored(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @callee_stored, ptr @fn_slot, align 8
+; CHECK-NEXT: call fastcc void @callee_stored(ptr [[P]])
+; CHECK-NEXT: ret void
+;
store ptr @callee_stored, ptr @fn_slot
call fastcc void @callee_stored(ptr %p)
ret void
@@ -541,20 +736,29 @@ define amdgpu_kernel void @k_stored(ptr %p) {
; musttail forwarding of an argument blocks inreg promotion for that argument.
define internal fastcc void @tail_target(ptr %q) {
+; CHECK-LABEL: define internal fastcc void @tail_target(
+; CHECK-SAME: ptr [[Q:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_musttail_forward(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: musttail call fastcc void @tail_target(ptr [[P]])
+; CHECK-NEXT: ret void
+;
musttail call fastcc void @tail_target(ptr %p)
ret void
}
define amdgpu_kernel void @k_musttail_forward(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
-; CHECK: call fastcc void @callee_musttail_forward(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_musttail_forward(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_musttail_forward(ptr %p)
ret void
}
@@ -562,9 +766,14 @@ define amdgpu_kernel void @k_musttail_forward(ptr %p) {
; inlinehint and alwaysinline callees behave the same for inreg promotion when
; the callee remains an out-of-line call (the motivating Kokkos case).
-; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -573,15 +782,24 @@ define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
-; CHECK: call fastcc void @callee_inlinehint_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_inlinehint_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
+; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -590,13 +808,419 @@ define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
-; CHECK: call fastcc void @callee_alwaysinline_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_alwaysinline_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
ret void
}
+; readfirstlane produces an always-uniform value (per TargetTransformInfo), so a
+; pointer laundered through it is promoted even though its input is divergent.
+
+define internal fastcc void @callee_readfirstlane(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_readfirstlane(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_readfirstlane(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_readfirstlane(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @callee_readfirstlane(ptr inreg [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @callee_readfirstlane(ptr %puni)
+ ret void
+}
+
+; A single callee reached from two kernels through different uniform sources (a
+; kernel SGPR argument and a readfirstlane result) is still promoted.
+
+define internal fastcc void @multi_src(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @multi_src(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_sgpr(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_sgpr(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @multi_src(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_readfirstlane(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_readfirstlane(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @multi_src(ptr %puni)
+ ret void
+}
+
+; Negative counterpart of the multi-source shape: one call site passes a uniform
+; (readfirstlane) operand and the other passes a divergent one. A single
+; divergent source blocks promotion of the shared definition, so no inreg appears
+; on the definition or at either call site.
+
+define internal fastcc void @multi_src_mixed(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @multi_src_mixed(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_mixed_uniform(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_uniform(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @multi_src_mixed(ptr %puni)
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_mixed_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_divergent(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @multi_src_mixed(ptr %pdiv)
+ ret void
+}
+
+; Self-recursive callee that forwards its own pointer argument: the fixpoint must
+; terminate. Promotion is conservatively skipped because the recursive call site
+; forwards the argument back to itself (the cycle cannot be proven uniform here).
+
+define internal fastcc void @self_rec(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @self_rec(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 [[N1]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ %n1 = sub i32 %n, 1
+ call fastcc void @self_rec(ptr %p, i32 %n1)
+ br label %end
+end:
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_self_rec(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_self_rec(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 10)
+; CHECK-NEXT: ret void
+;
+ call fastcc void @self_rec(ptr %p, i32 10)
+ ret void
+}
+
+; Mutually recursive callees that forward the pointer around a cycle: likewise
+; must terminate and are conservatively left unpromoted.
+
+define internal fastcc void @mutual_a(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @mutual_a(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: call fastcc void @mutual_b(ptr [[P]], i32 [[N1]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ %n1 = sub i32 %n, 1
+ call fastcc void @mutual_b(ptr %p, i32 %n1)
+ br label %end
+end:
+ ret void
+}
+
+define internal fastcc void @mutual_b(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @mutual_b(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 [[N]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ call fastcc void @mutual_a(ptr %p, i32 %n)
+ br label %end
+end:
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_mutual(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mutual(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 5)
+; CHECK-NEXT: ret void
+;
+ call fastcc void @mutual_a(ptr %p, i32 5)
+ ret void
+}
+
+; An intermediate caller that is address-taken could be reached indirectly with a
+; divergent pointer, so a leaf reached only through it must NOT be promoted even
+; though the visible direct call is uniform.
+
+ at addr_slot = global ptr null
+
+define internal fastcc void @ci_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci_leaf(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define internal fastcc void @ci_mid(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci_mid(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci_leaf(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_ci(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ci(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @ci_mid, ptr @addr_slot, align 8
+; CHECK-NEXT: call fastcc void @ci_mid(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ store ptr @ci_mid, ptr @addr_slot
+ call fastcc void @ci_mid(ptr %p)
+ ret void
+}
+
+; The same shared leaf reached through two intermediates: one address-taken, one
+; clean. The address-taken path taints the leaf (a single unprovable call site
+; blocks the shared definition), so @ci2_leaf is NOT promoted. The clean
+; intermediate @ci2_mid_clean is still promoted on its own, because its only call
+; site is the uniform kernel call and promotion is decided per callee.
+
+ at addr_slot2 = global ptr null
+
+define internal fastcc void @ci2_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_leaf(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define internal fastcc void @ci2_mid_at(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_mid_at(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci2_leaf(ptr %p)
+ ret void
+}
+
+define internal fastcc void @ci2_mid_clean(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_mid_clean(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci2_leaf(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_ci2(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ci2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @ci2_mid_at, ptr @addr_slot2, align 8
+; CHECK-NEXT: call fastcc void @ci2_mid_at(ptr [[P]])
+; CHECK-NEXT: call fastcc void @ci2_mid_clean(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
+ store ptr @ci2_mid_at, ptr @addr_slot2
+ call fastcc void @ci2_mid_at(ptr %p)
+ call fastcc void @ci2_mid_clean(ptr %p)
+ ret void
+}
+
+; A vector-of-pointers argument is not a scalar pointer and is out of scope.
+
+define internal fastcc void @callee_vecptr(<2 x ptr> %p) {
+; CHECK-LABEL: define internal fastcc void @callee_vecptr(
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define amdgpu_kernel void @k_vecptr(<2 x ptr> %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_vecptr(
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_vecptr(<2 x ptr> [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_vecptr(<2 x ptr> %p)
+ ret void
+}
+
+; A musttail call requires the enclosing function's parameter ABI attributes to
+; match the call positionally, regardless of which values are forwarded. Promoting
+; even an unrelated uniform argument (@p, which is not passed to the musttail
+; call) would break that contract, so no argument of a function containing a
+; musttail call is promoted.
+
+declare fastcc void @mt_abi_target(ptr, ptr)
+
+define internal fastcc void @callee_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define internal fastcc void @callee_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: musttail call fastcc void @mt_abi_target(ptr [[Q]], ptr [[Q]])
+; CHECK-NEXT: ret void
+;
+ musttail call fastcc void @mt_abi_target(ptr %q, ptr %q)
+ ret void
+}
+
+define amdgpu_kernel void @k_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_musttail_abi(ptr [[P]], ptr [[Q]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_musttail_abi(ptr %p, ptr %q)
+ ret void
+}
+
+; A freeze between the argument and a private addrspacecast must not hide the
+; private reinterpretation from the callee-side guard.
+
+define internal fastcc void @callee_freeze_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_freeze_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = freeze ptr [[P]]
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[F]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
+ %f = freeze ptr %p
+ %q = addrspacecast ptr %f to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_freeze_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_freeze_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_freeze_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_freeze_private(ptr %p)
+ ret void
+}
+
+; ptrmask likewise forwards the pointer; a private cast behind it must still be
+; detected by the callee-side guard.
+
+define internal fastcc void @callee_ptrmask_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_ptrmask_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[M:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -16)
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[M]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
+ %m = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -16)
+ %q = addrspacecast ptr %m to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_ptrmask_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ptrmask_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_ptrmask_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_ptrmask_private(ptr %p)
+ ret void
+}
+
attributes #0 = { inlinehint }
attributes #1 = { alwaysinline }
declare i32 @llvm.amdgcn.workitem.id.x()
+declare ptr @llvm.amdgcn.readfirstlane.p0(ptr)
+declare ptr @llvm.ptrmask.p0.i64(ptr, i64)
More information about the llvm-commits
mailing list