[llvm] [AMDGPU][NewPass] Attempt to promote uniform ptr arguments to inreg (PR #210410)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 10:50:44 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/210410
>From 88efb481bc1916f8de55e04907aeb0197b0aa491 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 17 Jul 2026 14:01:47 -0500
Subject: [PATCH 01/10] try to promote uniform ptr args to SGPRs
---
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 ++++++-
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 +-
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 541 ++++++++++++++++++
4 files changed, 775 insertions(+), 33 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 630ffad96e451..4a9d369076b4d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,8 +14,16 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -30,6 +38,15 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1434,6 +1451,186 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
+ AMDGPUInformationCache &InfoCache) {
+ const DataLayout &DL = M.getDataLayout();
+ TargetMachine &TM = InfoCache.TM;
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ const UniformityInfo *UI =
+ InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
+ *Caller);
+ if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged)
+ InfoCache.invalidateAnalyses();
+ }
+ }
+ return Changed;
+}
+
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1667,7 +1864,10 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- return A.run() == ChangeStatus::CHANGED;
+ bool PromoteChanged =
+ promoteUniformPointerArgsToInReg(M, AG, InfoCache);
+ bool AttChanged = A.run() == ChangeStatus::CHANGED;
+ return AttChanged || PromoteChanged;
}
} // namespace
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index cf4cb5f644bf9..ccb2ed7a54121 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,6 +298,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 5, i32 6}
-; CHECK: [[META1]] = !{}
+; CHECK: [[META0]] = !{i32 2, i32 10}
+; CHECK: [[META1]] = !{i32 5, i32 6}
+; CHECK: [[META2]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index f9edbd070ae7c..c95e6537fd0ab 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
new file mode 100644
index 0000000000000..81e85cd211a40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -0,0 +1,541 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+
+; A uniform pointer argument of an internal function, passed from a kernel,
+; is promoted to inreg (SGPR) on both the definition and the call site.
+
+; CHECK-LABEL: define internal fastcc void @callee_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
+; CHECK: call fastcc void @callee_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; A divergent pointer operand (derived from the workitem id) must NOT be
+; promoted, because inreg would drop all but one lane's value.
+
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_divergent(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
+; CHECK: call fastcc void @callee_divergent(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_divergent(ptr %pdiv)
+ ret void
+}
+
+; Private pointers name lane-private storage. Even if the pointer value itself
+; is uniform, the callee must not learn that the pointee is wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_private(
+; CHECK: call fastcc void @callee_private(ptr addrspace(5) %a)
+ %a = alloca i32, addrspace(5)
+ call fastcc void @callee_private(ptr addrspace(5) %a)
+ ret void
+}
+
+; A flat pointer derived from private memory carries the same risk.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_flat_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_private(
+; CHECK: call fastcc void @callee_flat_private(ptr %f)
+ %a = alloca i32, addrspace(5)
+ %f = addrspacecast ptr addrspace(5) %a to ptr
+ call fastcc void @callee_flat_private(ptr %f)
+ ret void
+}
+
+; A flat pointer that may be private on one path of a phi/select must not be
+; promoted, even though the other path is a benign global pointer.
+
+; CHECK-LABEL: define internal fastcc void @callee_phi_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_phi_private(ptr %p) {
+ store i32 1, ptr %p
+ ret void
+}
+
+ at gvar = addrspace(1) global i32 0
+
+define amdgpu_kernel void @k_phi_private(i1 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
+; CHECK: call fastcc void @callee_phi_private(ptr %sel)
+ %a = alloca i32, addrspace(5)
+ %fa = addrspacecast ptr addrspace(5) %a to ptr
+ %fg = addrspacecast ptr addrspace(1) @gvar to ptr
+ %sel = select i1 %c, ptr %fa, ptr %fg
+ call fastcc void @callee_phi_private(ptr %sel)
+ ret void
+}
+
+; A private pointer reached through a long getelementptr chain stays in the
+; private address space, so the address-space check still blocks it.
+
+; CHECK-LABEL: define internal fastcc void @callee_deep_private(
+; CHECK-SAME: ptr addrspace(5) %p
+; CHECK-NOT: ptr addrspace(5) inreg
+define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
+ store i32 1, ptr addrspace(5) %p
+ ret void
+}
+
+define amdgpu_kernel void @k_deep_private() {
+; CHECK-LABEL: define amdgpu_kernel void @k_deep_private(
+; CHECK: call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ %a = alloca [64 x i32], addrspace(5)
+ %g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
+ %g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
+ %g3 = getelementptr i32, ptr addrspace(5) %g2, i32 1
+ %g4 = getelementptr i32, ptr addrspace(5) %g3, i32 1
+ %g5 = getelementptr i32, ptr addrspace(5) %g4, i32 1
+ %g6 = getelementptr i32, ptr addrspace(5) %g5, i32 1
+ %g7 = getelementptr i32, ptr addrspace(5) %g6, i32 1
+ %g8 = getelementptr i32, ptr addrspace(5) %g7, i32 1
+ %g9 = getelementptr i32, ptr addrspace(5) %g8, i32 1
+ %g10 = getelementptr i32, ptr addrspace(5) %g9, i32 1
+ %g11 = getelementptr i32, ptr addrspace(5) %g10, i32 1
+ %g12 = getelementptr i32, ptr addrspace(5) %g11, i32 1
+ call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ ret void
+}
+
+; Callee-side guard: even a uniform flat pointer must not be promoted if the
+; callee reinterprets it as private (scratch), because that extracts a
+; lane-relative offset that is not wave-uniform.
+
+; CHECK-LABEL: define internal fastcc void @callee_casts_private(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_casts_private(ptr %p) {
+ %q = addrspacecast ptr %p to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_casts_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
+; CHECK: call fastcc void @callee_casts_private(ptr %p)
+ call fastcc void @callee_casts_private(ptr %p)
+ ret void
+}
+
+; Positive control: a flat load/store through the argument is the common, safe
+; use and must still be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_flat_load(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_flat_load(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_flat_load(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
+; CHECK: call fastcc void @callee_flat_load(ptr inreg %p)
+ call fastcc void @callee_flat_load(ptr %p)
+ ret void
+}
+
+; SGPR dword budget (default 8): only the first four pointer arguments fit;
+; the fifth is left in VGPRs.
+
+; CHECK-LABEL: define internal fastcc void @callee_budget(
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+ ret void
+}
+
+define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
+ ptr %p4) {
+; CHECK-LABEL: define amdgpu_kernel void @k_budget(
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+ call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
+ ret void
+}
+
+; TTI cross-check: a flat load is NeverUniform in GCNTTI even when the address
+; is wave-uniform, so the operand must not be promoted.
+
+; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_tti_flatload(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at gptr = addrspace(1) global ptr null
+
+define amdgpu_kernel void @k_tti_flatload() {
+; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload(
+; CHECK: call fastcc void @callee_tti_flatload(ptr %p)
+ %flatg = addrspacecast ptr addrspace(1) @gptr to ptr
+ %p = load ptr, ptr %flatg
+ call fastcc void @callee_tti_flatload(ptr %p)
+ ret void
+}
+
+; External linkage: all call sites are not necessarily visible, so the ABI
+; must not be changed.
+
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define fastcc void @callee_external(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_external(ptr %p) {
+ call fastcc void @callee_external(ptr %p)
+ ret void
+}
+
+; Address-taken internal function: an indirect call we cannot see could pass a
+; divergent value, so do not promote.
+
+; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_addrtaken(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+ at fnptr = global ptr null
+
+define amdgpu_kernel void @k_addrtaken(ptr %p) {
+ store ptr @callee_addrtaken, ptr @fnptr
+ call fastcc void @callee_addrtaken(ptr %p)
+ ret void
+}
+
+; Arguments with ABI-affecting attributes must not be promoted.
+
+%struct.Foo = type { i32 }
+
+; CHECK-LABEL: define internal fastcc void @callee_byref(
+; CHECK-SAME: ptr byref(%struct.Foo) %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_byref(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_byref(
+; CHECK: call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc ptr @callee_returned(
+; CHECK-SAME: ptr returned %p
+; CHECK-NOT: ptr inreg
+define internal fastcc ptr @callee_returned(ptr returned %p) {
+ ret ptr %p
+}
+
+define amdgpu_kernel void @k_returned(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_returned(
+; CHECK: call fastcc ptr @callee_returned(ptr returned %p)
+ %r = call fastcc ptr @callee_returned(ptr returned %p)
+ store ptr %r, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
+; CHECK-SAME: ptr swiftasync %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_swiftasync(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
+; CHECK: call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_hidden(
+; CHECK-SAME: ptr "amdgpu-hidden-argument" %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
+ ret void
+}
+
+define amdgpu_kernel void @k_hidden(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
+; CHECK: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+ ret void
+}
+
+; Non-pointer uniform arguments are out of scope for this pass.
+
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 %n
+; CHECK-NOT: i32 inreg
+define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+ store i32 %n, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; The pointer is still promoted, the scalar is not.
+; CHECK: call fastcc void @callee_scalar(i32 %n, ptr inreg %p)
+ call fastcc void @callee_scalar(i32 %n, ptr %p)
+ ret void
+}
+
+; Mixed call sites: one uniform, one divergent. A single divergent operand must
+; block promotion, since the definition is shared by all callers.
+
+; CHECK-LABEL: define internal fastcc void @callee_mixed(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_mixed(ptr %p) {
+ %v = load float, ptr %p
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
+; CHECK: call fastcc void @callee_mixed(ptr %p)
+ call fastcc void @callee_mixed(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
+; CHECK: call fastcc void @callee_mixed(ptr %pdiv)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_mixed(ptr %pdiv)
+ ret void
+}
+
+; Multi-hop chain: uniformity propagates from the kernel through each internal
+; function to a fixpoint, so every hop's closure pointer is promoted.
+
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_leaf(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_mid(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_mid(ptr %p) {
+; CHECK: call fastcc void @chain_leaf(ptr inreg %p)
+ call fastcc void @chain_leaf(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @chain_top(ptr %p) {
+; CHECK: call fastcc void @chain_mid(ptr inreg %p)
+ call fastcc void @chain_mid(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_chain(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_chain(
+; CHECK: call fastcc void @chain_top(ptr inreg %p)
+ call fastcc void @chain_top(ptr %p)
+ ret void
+}
+
+; Fixpoint must converge regardless of the order the functions appear in the
+; module (callees defined before/after their callers) and across a diamond.
+
+; CHECK-LABEL: define internal fastcc void @scram_a(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_a(ptr %p) {
+ call fastcc void @scram_b(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_c(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_c(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @scram_b(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @scram_b(ptr %p) {
+ call fastcc void @scram_c(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_scram(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scram(
+; CHECK: call fastcc void @scram_a(ptr inreg %p)
+ call fastcc void @scram_a(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_bot(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_bot(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_l(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_l(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_r(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_r(ptr %p) {
+ call fastcc void @diam_bot(ptr %p)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @diam_top(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @diam_top(ptr %p) {
+ call fastcc void @diam_l(ptr %p)
+ call fastcc void @diam_r(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_diam(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_diam(
+; CHECK: call fastcc void @diam_top(ptr inreg %p)
+ call fastcc void @diam_top(ptr %p)
+ ret void
+}
+
+; An argument that is already inreg must be left untouched (no double attribute,
+; no crash).
+
+; CHECK-LABEL: define internal fastcc void @callee_already(
+; CHECK-SAME: ptr inreg %p
+define internal fastcc void @callee_already(ptr inreg %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_already(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_already(
+; CHECK: call fastcc void @callee_already(ptr inreg %p)
+ call fastcc void @callee_already(ptr inreg %p)
+ ret void
+}
+
+; Invoke call sites are not audited for inreg ABI consistency under exceptional
+; control flow, so promotion is skipped.
+
+; CHECK-LABEL: define internal fastcc void @callee_invoke(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_invoke(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
+; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
+; CHECK: invoke fastcc void @callee_invoke(ptr %p)
+ invoke fastcc void @callee_invoke(ptr %p) to label %cont unwind label %lpad
+
+cont:
+ ret void
+
+lpad:
+ %tok = landingpad { ptr, i32 }
+ cleanup
+ ret void
+}
+
+; Indirect call through a bitcast of the function pointer: the callee is not a
+; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
+
+; CHECK-LABEL: define internal fastcc void @callee_bitcast(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_bitcast(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_bitcast(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
+; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+ %fn = bitcast ptr @callee_bitcast to ptr
+ call void %fn(ptr %p)
+ ret void
+}
+
+; A function stored into a global (non-call use) is not eligible even if there
+; is also a direct call the pass can see.
+
+; CHECK-LABEL: define internal fastcc void @callee_stored(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_stored(ptr %p) {
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+ at fn_slot = global ptr null
+
+define amdgpu_kernel void @k_stored(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_stored(
+; CHECK: call fastcc void @callee_stored(ptr %p)
+ store ptr @callee_stored, ptr @fn_slot
+ call fastcc void @callee_stored(ptr %p)
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
>From 45c2507ee5ef8ec6ac8e61e9bffc4a2cb8e32102 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 13:55:38 -0500
Subject: [PATCH 02/10] create new pass to promote uniform ptr args
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 5 +
llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 202 +-------------
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 251 ++++++++++++++++++
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 2 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/test/CodeGen/AMDGPU/aa-as-infer.ll | 57 ++--
.../AMDGPU/attributor-noalias-addrspace.ll | 8 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 45 +++-
9 files changed, 336 insertions(+), 236 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c72fa69aa1419..f77206cf6b63f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -314,6 +314,11 @@ struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
bool GlobalOpt;
};
+struct AMDGPUPromoteUniformArgsPass
+ : PassInfoMixin<AMDGPUPromoteUniformArgsPass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &);
extern char &AMDGPULowerExecSyncLegacyPassID;
ModulePass *createAMDGPULowerExecSyncLegacyPass();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index 4a9d369076b4d..630ffad96e451 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -14,16 +14,8 @@
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/IPO/Attributor.h"
#include <cstdint>
@@ -38,15 +30,6 @@ static cl::opt<unsigned> IndirectCallSpecializationThreshold(
"A threshold controls whether an indirect call will be specialized"),
cl::init(3));
-STATISTIC(NumPromotedInRegArgs,
- "Number of uniform pointer arguments promoted to inreg");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
-
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
enum ImplicitArgumentPositions {
@@ -1451,186 +1434,6 @@ struct AAAMDGPUMinAGPRAlloc
const char AAAMDGPUMinAGPRAlloc::ID = 0;
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
-}
-
-static bool isEligibleInRegUniformCallee(const Function &F) {
- if (F.isDeclaration() || F.isVarArg())
- return false;
- if (!F.hasLocalLinkage() || F.hasAddressTaken())
- return false;
- switch (F.getCallingConv()) {
- case CallingConv::C:
- case CallingConv::Fast:
- break;
- default:
- return false;
- }
- for (const User *U : F.users()) {
- const auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall() || isa<InvokeInst>(CB))
- return false;
- }
- if (F.user_empty())
- return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
- return true;
-}
-
-static bool mayBePrivateDerivedPointer(const Value *V) {
- assert(V->getType()->isPointerTy());
- if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
-
- SmallVector<const Value *, 8> Objects;
- getUnderlyingObjects(V, Objects);
- for (const Value *Obj : Objects) {
- if (isa<AllocaInst>(Obj))
- return true;
- if (Obj->getType()->isPointerTy() &&
- Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- }
- return false;
-}
-
-static bool calleeCastsArgToPrivate(const Argument &A) {
- SmallVector<const Value *, 16> Worklist;
- SmallPtrSet<const Value *, 16> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
- if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(ASC);
- continue;
- }
- if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
- if (II->getType()->getPointerAddressSpace() ==
- AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(II);
- }
- continue;
- }
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
- Worklist.push_back(U);
- }
- }
- return false;
-}
-
-static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall())
- return false;
- if (isa<InvokeInst>(CB))
- return false;
- Calls.push_back(CB);
- }
- return !Calls.empty();
-}
-
-static bool promoteUniformPointerArgsToInReg(Module &M, AnalysisGetter &AG,
- AMDGPUInformationCache &InfoCache) {
- const DataLayout &DL = M.getDataLayout();
- TargetMachine &TM = InfoCache.TM;
- bool Changed = false;
- bool RoundChanged = true;
- while (RoundChanged) {
- RoundChanged = false;
- for (Function &F : M) {
- if (!isEligibleInRegUniformCallee(F))
- continue;
-
- SmallVector<CallBase *, 8> Calls;
- if (!collectCallSites(F, Calls))
- continue;
-
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
- bool FuncChanged = false;
- for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
- continue;
- if (calleeCastsArgToPrivate(A))
- continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
- continue;
- }
-
- bool AllUniform = true;
- for (CallBase *CB : Calls) {
- Value *ArgOp = CB->getArgOperand(A.getArgNo());
- if (mayBePrivateDerivedPointer(ArgOp)) {
- AllUniform = false;
- break;
- }
-
- Function *Caller = CB->getFunction();
- const UniformityInfo *UI =
- InfoCache.getAnalysisResultForFunction<UniformityInfoAnalysis>(
- *Caller);
- if (UI && UI->isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
- AllUniform = false;
- break;
- }
-
- TargetTransformInfo TTI = TM.getTargetTransformInfo(*Caller);
- if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
- AllUniform = false;
- break;
- }
- }
- if (!AllUniform)
- continue;
-
- A.addAttr(Attribute::InReg);
- for (CallBase *CB : Calls)
- CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
- ++NumPromotedInRegArgs;
- FuncChanged = Changed = RoundChanged = true;
- }
-
- if (FuncChanged)
- InfoCache.invalidateAnalyses();
- }
- }
- return Changed;
-}
-
/// An abstract attribute to propagate the function attribute
/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
struct AAAMDGPUClusterDims
@@ -1864,10 +1667,7 @@ static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
}
}
- bool PromoteChanged =
- promoteUniformPointerArgsToInReg(M, AG, InfoCache);
- bool AttChanged = A.run() == ChangeStatus::CHANGED;
- return AttChanged || PromoteChanged;
+ return A.run() == ChangeStatus::CHANGED;
}
} // namespace
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index d052f3c73920c..6a90ff2522870 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -17,6 +17,7 @@
#define MODULE_PASS(NAME, CREATE_PASS)
#endif
MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
+MODULE_PASS("amdgpu-promote-uniform-args", AMDGPUPromoteUniformArgsPass())
MODULE_PASS("amdgpu-export-kernel-runtime-handles", AMDGPUExportKernelRuntimeHandlesPass())
MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
AMDGPULowerBufferFatPointersPass(*this))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
new file mode 100644
index 0000000000000..dd689d4667ad5
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -0,0 +1,251 @@
+//===-- AMDGPUPromoteUniformArgs.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// For non-entry AMDGPU functions, pointer arguments are passed in VGPRs unless
+// marked \c inreg. When such an argument is actually uniform across the
+// wavefront at every visible call site, passing it in VGPRs forces every lane
+// to carry the same value and can inflate register pressure. This pass promotes
+// provably-uniform pointer arguments of internal callees to \c inreg (SGPR
+// passing) on the definition and at each direct call site.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-promote-uniform-args"
+
+STATISTIC(NumPromotedInRegArgs,
+ "Number of uniform pointer arguments promoted to inreg");
+STATISTIC(NumPromotedInRegFuncs,
+ "Number of functions with a promoted uniform pointer argument");
+STATISTIC(NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
+
+static cl::opt<bool> EnablePromoteUniformPointerArgs(
+ "amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
+ cl::desc("Promote provably uniform internal pointer arguments to inreg"));
+
+static cl::opt<unsigned> UniformArgSGPRDwordBudget(
+ "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
+ cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
+
+namespace {
+
+static bool hasBlockingInRegArgAttr(const Argument &A) {
+ return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
+ A.hasAttribute(Attribute::ByRef) ||
+ A.hasAttribute(Attribute::StructRet) ||
+ A.hasAttribute(Attribute::InAlloca) ||
+ A.hasAttribute(Attribute::Preallocated) ||
+ A.hasAttribute(Attribute::Nest) ||
+ A.hasAttribute(Attribute::Returned) ||
+ A.hasAttribute(Attribute::SwiftError) ||
+ A.hasAttribute(Attribute::SwiftSelf) ||
+ A.hasAttribute(Attribute::SwiftAsync) ||
+ A.hasAttribute("amdgpu-hidden-argument");
+}
+
+static bool isEligibleInRegUniformCallee(const Function &F) {
+ if (F.isDeclaration() || F.isVarArg())
+ return false;
+ if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ return false;
+ switch (F.getCallingConv()) {
+ case CallingConv::C:
+ case CallingConv::Fast:
+ break;
+ default:
+ return false;
+ }
+ for (const User *U : F.users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall() || isa<InvokeInst>(CB))
+ return false;
+ }
+ if (F.user_empty())
+ return false;
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+ return true;
+}
+
+static bool mayBePrivateDerivedPointer(const Value *V) {
+ assert(V->getType()->isPointerTy());
+ if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+
+ SmallVector<const Value *, 8> Objects;
+ getUnderlyingObjects(V, Objects);
+ for (const Value *Obj : Objects) {
+ if (isa<AllocaInst>(Obj))
+ return true;
+ if (Obj->getType()->isPointerTy() &&
+ Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ }
+ return false;
+}
+
+static bool calleeCastsArgToPrivate(const Argument &A) {
+ SmallVector<const Value *, 16> Worklist;
+ SmallPtrSet<const Value *, 16> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
+ if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(ASC);
+ continue;
+ }
+ if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ if (II->getType()->getPointerAddressSpace() ==
+ AMDGPUAS::PRIVATE_ADDRESS)
+ return true;
+ Worklist.push_back(II);
+ }
+ continue;
+ }
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ Worklist.push_back(U);
+ }
+ }
+ return false;
+}
+
+static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ return false;
+ if (CB->isMustTailCall())
+ return false;
+ if (isa<InvokeInst>(CB))
+ return false;
+ Calls.push_back(CB);
+ }
+ return !Calls.empty();
+}
+
+static bool promoteUniformPointerArgsToInReg(Module &M,
+ ModuleAnalysisManager &AM) {
+ auto &FAM =
+ AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ const DataLayout &DL = M.getDataLayout();
+ bool Changed = false;
+ bool RoundChanged = true;
+ while (RoundChanged) {
+ RoundChanged = false;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
+
+ SmallVector<CallBase *, 8> Calls;
+ if (!collectCallSites(F, Calls))
+ continue;
+
+ unsigned UsedDwords = 0;
+ for (Argument &A : F.args())
+ if (A.hasAttribute(Attribute::InReg))
+ UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ continue;
+ if (calleeCastsArgToPrivate(A))
+ continue;
+
+ unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
+ if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
+ ++NumSkippedDueToInRegBudget;
+ continue;
+ }
+
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ Value *ArgOp = CB->getArgOperand(A.getArgNo());
+ if (mayBePrivateDerivedPointer(ArgOp)) {
+ AllUniform = false;
+ break;
+ }
+
+ Function *Caller = CB->getFunction();
+ UniformityInfo &UI =
+ FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
+ AllUniform = false;
+ break;
+ }
+
+ const TargetTransformInfo &TTI =
+ FAM.getResult<TargetIRAnalysis>(*Caller);
+ if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ AllUniform = false;
+ break;
+ }
+ }
+ if (!AllUniform)
+ continue;
+
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ UsedDwords += Need;
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = RoundChanged = true;
+ }
+
+ if (FuncChanged) {
+ ++NumPromotedInRegFuncs;
+ FAM.invalidate(F, PreservedAnalyses::none());
+ SmallPtrSet<Function *, 8> InvalidatedCallers;
+ for (CallBase *CB : Calls) {
+ Function *Caller = CB->getFunction();
+ if (Caller != &F && InvalidatedCallers.insert(Caller).second)
+ FAM.invalidate(*Caller, PreservedAnalyses::none());
+ }
+ }
+ }
+ }
+ return Changed;
+}
+
+} // namespace
+
+PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ return PreservedAnalyses::all();
+ return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
+ : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 8c2732630e2e6..ce2aaacfb6969 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1101,6 +1101,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
ThinOrFullLTOPhase Phase) {
if (Level != OptimizationLevel::O0) {
if (!isLTOPreLink(Phase)) {
+ if (Level > OptimizationLevel::O1 && getTargetTriple().isAMDGCN())
+ MPM.addPass(AMDGPUPromoteUniformArgsPass());
if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
AMDGPUAttributorOptions Opts;
MPM.addPass(AMDGPUAttributorPass(*this, Opts, Phase));
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..40afd303343f9 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -102,6 +102,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUPrintfRuntimeBinding.cpp
AMDGPUPromoteAlloca.cpp
AMDGPUPromoteKernelArguments.cpp
+ AMDGPUPromoteUniformArgs.cpp
AMDGPURegBankCombiner.cpp
AMDGPURegBankLegalize.cpp
AMDGPURegBankLegalizeHelper.cpp
diff --git a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
index ccb2ed7a54121..cf4cb5f644bf9 100644
--- a/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
+++ b/llvm/test/CodeGen/AMDGPU/aa-as-infer.ll
@@ -90,21 +90,21 @@ define void @call_volatile_load_store_as_4(ptr addrspace(4) %p1, ptr addrspace(4
define internal void @can_infer_cmpxchg(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_cmpxchg(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4, !noalias.addrspace [[META0:![0-9]+]]
+; CHECK-NEXT: [[CMPXCHG_0:%.*]] = cmpxchg ptr addrspace(1) [[TMP1]], i32 0, i32 4 monotonic monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_1:%.*]] = cmpxchg ptr addrspace(1) [[TMP2]], i32 0, i32 5 acq_rel monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_2:%.*]] = cmpxchg ptr addrspace(1) [[TMP3]], i32 0, i32 6 acquire monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_3:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i32 0, i32 7 release monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_4:%.*]] = cmpxchg ptr addrspace(1) [[TMP5]], i32 0, i32 8 seq_cst monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[CMPXCHG_5:%.*]] = cmpxchg weak ptr addrspace(1) [[TMP6]], i32 0, i32 9 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_6:%.*]] = cmpxchg volatile ptr [[WORD]], i32 0, i32 10 seq_cst monotonic, align 4
+; CHECK-NEXT: [[CMPXCHG_7:%.*]] = cmpxchg weak volatile ptr [[WORD]], i32 0, i32 11 syncscope("singlethread") seq_cst monotonic, align 4
; CHECK-NEXT: ret void
;
%cmpxchg.0 = cmpxchg ptr %word, i32 0, i32 4 monotonic monotonic, align 4
@@ -144,27 +144,27 @@ define internal void @can_not_infer_cmpxchg(ptr %word) {
define internal void @can_infer_atomicrmw(ptr %word) {
; CHECK-LABEL: define internal void @can_infer_atomicrmw(
-; CHECK-SAME: ptr inreg [[WORD:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[WORD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XCHG:%.*]] = atomicrmw xchg ptr addrspace(1) [[TMP1]], i32 12 monotonic, align 4
; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_ADD:%.*]] = atomicrmw add ptr addrspace(1) [[TMP2]], i32 13 monotonic, align 4
; CHECK-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_SUB:%.*]] = atomicrmw sub ptr addrspace(1) [[TMP3]], i32 14 monotonic, align 4
; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_AND:%.*]] = atomicrmw and ptr addrspace(1) [[TMP4]], i32 15 monotonic, align 4
; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_NAND:%.*]] = atomicrmw nand ptr addrspace(1) [[TMP5]], i32 16 monotonic, align 4
; CHECK-NEXT: [[TMP6:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_OR:%.*]] = atomicrmw or ptr addrspace(1) [[TMP6]], i32 17 monotonic, align 4
; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_XOR:%.*]] = atomicrmw xor ptr addrspace(1) [[TMP7]], i32 18 monotonic, align 4
; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_MAX:%.*]] = atomicrmw max ptr addrspace(1) [[TMP8]], i32 19 monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_MIN:%.*]] = atomicrmw volatile min ptr [[WORD]], i32 20 monotonic, align 4
; CHECK-NEXT: [[TMP10:%.*]] = addrspacecast ptr [[WORD]] to ptr addrspace(1)
-; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
-; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4, !noalias.addrspace [[META0]]
+; CHECK-NEXT: [[ATOMICRMW_UMAX:%.*]] = atomicrmw umax ptr addrspace(1) [[TMP10]], i32 21 syncscope("singlethread") monotonic, align 4
+; CHECK-NEXT: [[ATOMICRMW_UMIN:%.*]] = atomicrmw volatile umin ptr [[WORD]], i32 22 syncscope("singlethread") monotonic, align 4
; CHECK-NEXT: ret void
;
%atomicrmw.xchg = atomicrmw xchg ptr %word, i32 12 monotonic, align 4
@@ -215,13 +215,13 @@ define void @foo(ptr addrspace(3) %val) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr addrspace(3) [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[VAL_CAST:%.*]] = addrspacecast ptr addrspace(3) [[VAL]] to ptr
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_cmpxchg(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_cmpxchg(ptr [[VAL_CAST]])
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g1 to ptr))
-; CHECK-NEXT: call void @can_infer_atomicrmw(ptr inreg addrspacecast (ptr addrspace(1) @g2 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
+; CHECK-NEXT: call void @can_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g1 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr addrspacecast (ptr addrspace(1) @g2 to ptr))
; CHECK-NEXT: call void @can_not_infer_atomicrmw(ptr [[VAL_CAST]])
@@ -284,7 +284,7 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
; CHECK-NEXT: [[P3_CAST:%.*]] = addrspacecast ptr addrspace(3) [[P3]] to ptr
; CHECK-NEXT: [[B:%.*]] = icmp eq i32 [[VAL]], 0
; CHECK-NEXT: [[P:%.*]] = select i1 [[B]], ptr [[P1_CAST]], ptr [[P3_CAST]]
-; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META2:![0-9]+]], !amdgpu.no.remote.memory [[META2]]
+; CHECK-NEXT: [[ATOMIC_ADD:%.*]] = atomicrmw add ptr [[P]], i32 1 syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]], !amdgpu.no.remote.memory [[META1]]
; CHECK-NEXT: ret void
;
%p1.cast = addrspacecast ptr addrspace(1) %p1 to ptr
@@ -298,7 +298,6 @@ define amdgpu_kernel void @kernel_argument_with_known_as(ptr addrspace(1) %p1, p
!0 = !{i32 5, i32 6}
!1 = !{}
;.
-; CHECK: [[META0]] = !{i32 2, i32 10}
-; CHECK: [[META1]] = !{i32 5, i32 6}
-; CHECK: [[META2]] = !{}
+; CHECK: [[META0]] = !{i32 5, i32 6}
+; CHECK: [[META1]] = !{}
;.
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
index c95e6537fd0ab..f9edbd070ae7c 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-noalias-addrspace.ll
@@ -480,7 +480,7 @@ bb.2.end:
define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_no_alias_addr_space_select(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[PTR4:%.*]] = select i1 [[COND1]], ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr)
; CHECK-NEXT: [[PTR5:%.*]] = select i1 [[COND2]], ptr [[PTR4]], ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr)
; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR5]], align 4, !noalias.addrspace [[META1:![0-9]+]]
@@ -516,7 +516,7 @@ define internal void @callee_no_alias_addr_space_select(ptr %ptr1, ptr %ptr2, pt
define internal void @callee_alias_addr_space_branch(ptr %ptr1, ptr %ptr2, ptr %ptr3, i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define internal void @callee_alias_addr_space_branch(
-; CHECK-SAME: ptr inreg [[PTR1:%.*]], ptr inreg [[PTR2:%.*]], ptr inreg [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], ptr [[PTR3:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: br i1 [[COND1]], label %[[BB_1_TRUE:.*]], label %[[BB_1_FALSE:.*]]
; CHECK: [[BB_1_TRUE]]:
; CHECK-NEXT: br label %[[BB_1_END:.*]]
@@ -578,8 +578,8 @@ define amdgpu_kernel void @kernal_call_func(i1 %cond1, i1 %cond2, i32 %val) #0 {
; CHECK-LABEL: define amdgpu_kernel void @kernal_call_func(
; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], i32 [[VAL:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[LPTR:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
-; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr inreg addrspacecast (ptr addrspace(1) @gptr to ptr), ptr inreg addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr inreg addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_no_alias_addr_space_select(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
+; CHECK-NEXT: call void @callee_alias_addr_space_branch(ptr addrspacecast (ptr addrspace(1) @gptr to ptr), ptr addrspacecast (ptr addrspace(4) @gptr2 to ptr), ptr addrspacecast (ptr addrspace(3) @gptr3 to ptr), i1 [[COND1]], i1 [[COND2]], i32 [[VAL]])
; CHECK-NEXT: ret void
;
%lptr = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 81e85cd211a40..1568fbb3e561f 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -511,7 +511,8 @@ define internal fastcc void @callee_bitcast(ptr %p) {
define amdgpu_kernel void @k_bitcast(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK: call {{.*}} @callee_bitcast(ptr {{.*}}%p)
+; CHECK: call void {{.*}}(ptr {{.*}}%p)
+; CHECK-NOT: inreg
%fn = bitcast ptr @callee_bitcast to ptr
call void %fn(ptr %p)
ret void
@@ -538,4 +539,44 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; inlinehint and alwaysinline callees behave the same for inreg promotion when
+; the callee remains an out-of-line call (the motivating Kokkos case).
+
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
+; CHECK: call fastcc void @callee_inlinehint_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
+; CHECK-SAME: ptr inreg {{.*}}%p
+define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
+ %g = getelementptr float, ptr %p, i32 %i
+ %v = load float, ptr %g
+ store float %v, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
+; CHECK: call fastcc void @callee_alwaysinline_uniform(ptr inreg %p, i32 %tid)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
+ ret void
+}
+
+attributes #0 = { inlinehint }
+attributes #1 = { alwaysinline }
+
declare i32 @llvm.amdgcn.workitem.id.x()
>From 97276e5e9b6c98e63b8d32bfedfd49f0d827c1bb Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Thu, 30 Jul 2026 14:23:53 -0500
Subject: [PATCH 03/10] code format
---
.../Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index dd689d4667ad5..9bf13cb6b58e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -38,8 +38,9 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
+STATISTIC(
+ NumSkippedDueToInRegBudget,
+ "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
@@ -158,8 +159,7 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
- auto &FAM =
- AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
@@ -200,8 +200,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
}
Function *Caller = CB->getFunction();
- UniformityInfo &UI =
- FAM.getResult<UniformityInfoAnalysis>(*Caller);
+ UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(*Caller);
if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
AllUniform = false;
break;
@@ -243,8 +242,9 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
} // namespace
PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
- ModuleAnalysisManager &AM) {
- if (!EnablePromoteUniformPointerArgs || !Triple(M.getTargetTriple()).isAMDGCN())
+ ModuleAnalysisManager &AM) {
+ if (!EnablePromoteUniformPointerArgs ||
+ !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
: PreservedAnalyses::all();
>From 5ede768d47f41ac6b3b9842a17f8d859ba6c8dac Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 31 Jul 2026 08:46:18 -0500
Subject: [PATCH 04/10] update hard coded list, musttail scan, preserve CFG
analysis, SGPR budget drop
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 87 ++++++++++---------
.../CodeGen/AMDGPU/promote-uniform-args.ll | 30 +++++--
2 files changed, 69 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 9bf13cb6b58e3..d29881c8e7965 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -23,12 +23,12 @@
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Analysis.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MathExtras.h"
using namespace llvm;
@@ -38,32 +38,21 @@ STATISTIC(NumPromotedInRegArgs,
"Number of uniform pointer arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-STATISTIC(
- NumSkippedDueToInRegBudget,
- "Number of uniform pointer arguments not promoted due to SGPR budget");
static cl::opt<bool> EnablePromoteUniformPointerArgs(
"amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
cl::desc("Promote provably uniform internal pointer arguments to inreg"));
-static cl::opt<unsigned> UniformArgSGPRDwordBudget(
- "amdgpu-uniform-args-sgpr-budget", cl::Hidden, cl::init(8),
- cl::desc("Max total SGPR dwords of inreg pointer arguments per function"));
-
namespace {
-static bool hasBlockingInRegArgAttr(const Argument &A) {
- return A.hasAttribute(Attribute::InReg) || A.hasAttribute(Attribute::ByVal) ||
- A.hasAttribute(Attribute::ByRef) ||
- A.hasAttribute(Attribute::StructRet) ||
- A.hasAttribute(Attribute::InAlloca) ||
- A.hasAttribute(Attribute::Preallocated) ||
- A.hasAttribute(Attribute::Nest) ||
- A.hasAttribute(Attribute::Returned) ||
- A.hasAttribute(Attribute::SwiftError) ||
- A.hasAttribute(Attribute::SwiftSelf) ||
- A.hasAttribute(Attribute::SwiftAsync) ||
- A.hasAttribute("amdgpu-hidden-argument");
+static bool canPromoteArgToInReg(const Argument &A) {
+ if (!A.getType()->isPointerTy() || A.hasInRegAttr())
+ return false;
+ if (A.hasPointeeInMemoryValueAttr() || A.hasNestAttr() ||
+ A.hasReturnedAttr() || A.hasSwiftSelfAttr() || A.hasSwiftErrorAttr() ||
+ A.hasAttribute(Attribute::SwiftAsync))
+ return false;
+ return !A.hasAttribute("amdgpu-hidden-argument");
}
static bool isEligibleInRegUniformCallee(const Function &F) {
@@ -87,14 +76,31 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
}
if (F.user_empty())
return false;
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return false;
return true;
}
+static bool argForwardedByMustTail(const Argument &A) {
+ SmallVector<const Value *, 8> Worklist;
+ SmallPtrSet<const Value *, 8> Visited;
+ Worklist.push_back(&A);
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ for (const User *U : V->users()) {
+ if (const auto *CB = dyn_cast<CallBase>(U)) {
+ if (CB->isMustTailCall())
+ return true;
+ continue;
+ }
+ if (isa<BitCastInst, GetElementPtrInst, AddrSpaceCastInst, PHINode,
+ SelectInst>(U))
+ Worklist.push_back(cast<Instruction>(U));
+ }
+ }
+ return false;
+}
+
static bool mayBePrivateDerivedPointer(const Value *V) {
assert(V->getType()->isPointerTy());
if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
@@ -160,7 +166,6 @@ static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- const DataLayout &DL = M.getDataLayout();
bool Changed = false;
bool RoundChanged = true;
while (RoundChanged) {
@@ -173,23 +178,14 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
if (!collectCallSites(F, Calls))
continue;
- unsigned UsedDwords = 0;
- for (Argument &A : F.args())
- if (A.hasAttribute(Attribute::InReg))
- UsedDwords += divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
-
bool FuncChanged = false;
for (Argument &A : F.args()) {
- if (hasBlockingInRegArgAttr(A) || !A.getType()->isPointerTy())
+ if (!canPromoteArgToInReg(A))
continue;
- if (calleeCastsArgToPrivate(A))
+ if (argForwardedByMustTail(A))
continue;
-
- unsigned Need = divideCeil(DL.getTypeSizeInBits(A.getType()), 32);
- if (UsedDwords + Need > UniformArgSGPRDwordBudget) {
- ++NumSkippedDueToInRegBudget;
+ if (calleeCastsArgToPrivate(A))
continue;
- }
bool AllUniform = true;
for (CallBase *CB : Calls) {
@@ -219,19 +215,21 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
A.addAttr(Attribute::InReg);
for (CallBase *CB : Calls)
CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- UsedDwords += Need;
++NumPromotedInRegArgs;
FuncChanged = Changed = RoundChanged = true;
}
if (FuncChanged) {
++NumPromotedInRegFuncs;
- FAM.invalidate(F, PreservedAnalyses::none());
+ PreservedAnalyses FuncPA;
+ // Attribute-only change; the CFG is unchanged.
+ FuncPA.preserveSet<CFGAnalyses>();
+ FAM.invalidate(F, FuncPA);
SmallPtrSet<Function *, 8> InvalidatedCallers;
for (CallBase *CB : Calls) {
Function *Caller = CB->getFunction();
if (Caller != &F && InvalidatedCallers.insert(Caller).second)
- FAM.invalidate(*Caller, PreservedAnalyses::none());
+ FAM.invalidate(*Caller, FuncPA);
}
}
}
@@ -246,6 +244,9 @@ PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
if (!EnablePromoteUniformPointerArgs ||
!Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
- return promoteUniformPointerArgsToInReg(M, AM) ? PreservedAnalyses::none()
- : PreservedAnalyses::all();
+ if (!promoteUniformPointerArgsToInReg(M, AM))
+ return PreservedAnalyses::all();
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 1568fbb3e561f..97b0476278369 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
@@ -172,11 +172,10 @@ define amdgpu_kernel void @k_flat_load(ptr %p) {
ret void
}
-; SGPR dword budget (default 8): only the first four pointer arguments fit;
-; the fifth is left in VGPRs.
+; Multiple uniform pointer arguments are all promoted opportunistically.
; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4
+; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
ret void
@@ -185,7 +184,7 @@ define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
ptr %p4) {
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr %p4)
+; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4)
call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
ret void
}
@@ -539,6 +538,27 @@ define amdgpu_kernel void @k_stored(ptr %p) {
ret void
}
+; musttail forwarding of an argument blocks inreg promotion for that argument.
+
+define internal fastcc void @tail_target(ptr %q) {
+ ret void
+}
+
+; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
+; CHECK-SAME: ptr %p
+; CHECK-NOT: ptr inreg
+define internal fastcc void @callee_musttail_forward(ptr %p) {
+ musttail call fastcc void @tail_target(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_musttail_forward(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
+; CHECK: call fastcc void @callee_musttail_forward(ptr %p)
+ call fastcc void @callee_musttail_forward(ptr %p)
+ ret void
+}
+
; inlinehint and alwaysinline callees behave the same for inreg promotion when
; the callee remains an out-of-line call (the motivating Kokkos case).
>From eed465b8167c2dbc4653775b452d4f76fc91cb5d Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 31 Jul 2026 11:33:04 -0500
Subject: [PATCH 05/10] code format
---
llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index d29881c8e7965..2d0e5b9060b78 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -22,8 +22,8 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Attributes.h"
#include "llvm/IR/Analysis.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
>From 4c3eb3888d99529b4b750a837787c1ea6a8c4360 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 14 Aug 2026 11:47:25 -0500
Subject: [PATCH 06/10] Replace UniformityAnalysis with a conservative trivial
uniformity check (TTI AlwaysUniform plus recursive propagation through
visible caller chains), and add a path-based cycle guard for recursive arg
forwarding.
Fix two correctness bugs before promotion:
- Skip any function containing a musttail call; adding inreg to any parameter breaks positional ABI matching, not just forwarded args.
- Follow freeze, ptrmask, and pointer-aliasing intrinsics when detecting private address-space casts in the callee.
additional tests for various promotion accept/reject use cases, tests for the musttail, freeze, and ptrmask cases.
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 156 ++--
.../CodeGen/AMDGPU/promote-uniform-args.ll | 882 +++++++++++++++---
2 files changed, 855 insertions(+), 183 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 2d0e5b9060b78..59d77698aae30 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -6,12 +6,12 @@
//
//===----------------------------------------------------------------------===//
//
-// For non-entry AMDGPU functions, pointer arguments are passed in VGPRs unless
-// marked \c inreg. When such an argument is actually uniform across the
-// wavefront at every visible call site, passing it in VGPRs forces every lane
-// to carry the same value and can inflate register pressure. This pass promotes
-// provably-uniform pointer arguments of internal callees to \c inreg (SGPR
-// passing) on the definition and at each direct call site.
+// Promote provably-uniform pointer arguments of internal callees to \c inreg
+// (SGPR passing) on the definition and at every direct call site. Uniformity is
+// established conservatively: an operand is uniform if it is an always-uniform
+// instruction (per TargetTransformInfo) or an argument that every visible
+// caller forwards from another uniform value. A full \c UniformityInfo-based
+// check is a planned follow-up.
//
//===----------------------------------------------------------------------===//
@@ -20,7 +20,6 @@
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Attributes.h"
@@ -39,8 +38,8 @@ STATISTIC(NumPromotedInRegArgs,
STATISTIC(NumPromotedInRegFuncs,
"Number of functions with a promoted uniform pointer argument");
-static cl::opt<bool> EnablePromoteUniformPointerArgs(
- "amdgpu-promote-uniform-pointer-args", cl::Hidden, cl::init(true),
+static cl::opt<bool> EnablePromoteUniformArgs(
+ "amdgpu-enable-promote-uniform-args", cl::Hidden, cl::init(true),
cl::desc("Promote provably uniform internal pointer arguments to inreg"));
namespace {
@@ -79,25 +78,14 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
return true;
}
-static bool argForwardedByMustTail(const Argument &A) {
- SmallVector<const Value *, 8> Worklist;
- SmallPtrSet<const Value *, 8> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *CB = dyn_cast<CallBase>(U)) {
+// musttail requires the enclosing function's parameter ABI (including inreg) to
+// match positionally, so skip promotion for any function containing one.
+static bool hasMustTailCall(const Function &F) {
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
if (CB->isMustTailCall())
return true;
- continue;
- }
- if (isa<BitCastInst, GetElementPtrInst, AddrSpaceCastInst, PHINode,
- SelectInst>(U))
- Worklist.push_back(cast<Instruction>(U));
- }
- }
return false;
}
@@ -134,38 +122,97 @@ static bool calleeCastsArgToPrivate(const Argument &A) {
continue;
}
if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- if (II->getIntrinsicID() == Intrinsic::amdgcn_addrspacecast_nonnull) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::amdgcn_addrspacecast_nonnull:
if (II->getType()->getPointerAddressSpace() ==
AMDGPUAS::PRIVATE_ADDRESS)
return true;
Worklist.push_back(II);
+ break;
+ case Intrinsic::ptrmask:
+ case Intrinsic::launder_invariant_group:
+ case Intrinsic::strip_invariant_group:
+ Worklist.push_back(II);
+ break;
+ default:
+ break;
}
continue;
}
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst>(U))
+ if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst, FreezeInst>(
+ U))
Worklist.push_back(U);
}
}
return false;
}
-static bool collectCallSites(Function &F, SmallVectorImpl<CallBase *> &Calls) {
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- return false;
- if (CB->isMustTailCall())
- return false;
- if (isa<InvokeInst>(CB))
- return false;
- Calls.push_back(CB);
+static bool isTriviallyUniformArg(
+ const Argument *Arg,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited);
+
+static bool isTriviallyUniformOperand(
+ const Value *V,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited) {
+ if (const auto *Arg = dyn_cast<Argument>(V))
+ return isTriviallyUniformArg(Arg, GetTTI, Visited);
+
+ if (const auto *I = dyn_cast<Instruction>(V)) {
+ const TargetTransformInfo &TTI = GetTTI(*I->getFunction());
+ return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
+ }
+
+ return false;
+}
+
+static bool isTriviallyUniformArg(
+ const Argument *Arg,
+ function_ref<TargetTransformInfo &(const Function &)> GetTTI,
+ SmallPtrSetImpl<const Argument *> &Visited) {
+ if (Arg->hasInRegAttr())
+ return true;
+
+ const Function *F = Arg->getParent();
+ if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+ return AMDGPU::isArgPassedInSGPR(Arg);
+
+ // Only direct-call-only internal callees have fully visible callers; an
+ // indirect or address-taken caller could pass a divergent value we never see.
+ if (!isEligibleInRegUniformCallee(*F))
+ return false;
+
+ // Path-based cycle guard: a (mutually) recursive chain that forwards the
+ // argument back to itself cannot be proven here and must terminate. The
+ // argument is removed on the way out so a diamond-shaped call graph can still
+ // reach the same argument through an independent path.
+ if (!Visited.insert(Arg).second)
+ return false;
+
+ bool HasCallSite = false;
+ bool AllUniform = true;
+ for (const User *U : F->users()) {
+ const auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != F)
+ continue;
+ HasCallSite = true;
+ if (!isTriviallyUniformOperand(CB->getArgOperand(Arg->getArgNo()), GetTTI,
+ Visited)) {
+ AllUniform = false;
+ break;
+ }
}
- return !Calls.empty();
+ Visited.erase(Arg);
+ return HasCallSite && AllUniform;
}
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto GetTTI = [&FAM](const Function &F) -> TargetTransformInfo & {
+ return FAM.getResult<TargetIRAnalysis>(const_cast<Function &>(F));
+ };
bool Changed = false;
bool RoundChanged = true;
while (RoundChanged) {
@@ -174,16 +221,26 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
if (!isEligibleInRegUniformCallee(F))
continue;
+ // Promoting any argument changes F's ABI attributes, which a musttail
+ // call in F's body would then no longer match positionally. Skip
+ // conservatively.
+ if (hasMustTailCall(F))
+ continue;
+
+ // isEligibleInRegUniformCallee guarantees every user is a direct,
+ // non-musttail, non-invoke call to F, so we can just gather them.
SmallVector<CallBase *, 8> Calls;
- if (!collectCallSites(F, Calls))
+ for (User *U : F.users())
+ if (auto *CB = dyn_cast<CallBase>(U);
+ CB && CB->getCalledFunction() == &F)
+ Calls.push_back(CB);
+ if (Calls.empty())
continue;
bool FuncChanged = false;
for (Argument &A : F.args()) {
if (!canPromoteArgToInReg(A))
continue;
- if (argForwardedByMustTail(A))
- continue;
if (calleeCastsArgToPrivate(A))
continue;
@@ -195,16 +252,8 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
break;
}
- Function *Caller = CB->getFunction();
- UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(*Caller);
- if (UI.isDivergentAtUse(CB->getArgOperandUse(A.getArgNo()))) {
- AllUniform = false;
- break;
- }
-
- const TargetTransformInfo &TTI =
- FAM.getResult<TargetIRAnalysis>(*Caller);
- if (TTI.getValueUniformity(ArgOp) == ValueUniformity::NeverUniform) {
+ SmallPtrSet<const Argument *, 4> VisitedArgs;
+ if (!isTriviallyUniformOperand(ArgOp, GetTTI, VisitedArgs)) {
AllUniform = false;
break;
}
@@ -241,8 +290,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
ModuleAnalysisManager &AM) {
- if (!EnablePromoteUniformPointerArgs ||
- !Triple(M.getTargetTriple()).isAMDGCN())
+ if (!EnablePromoteUniformArgs || !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
if (!promoteUniformPointerArgsToInReg(M, AM))
return PreservedAnalyses::all();
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 97b0476278369..64bff2fc40644 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,11 +1,17 @@
-; RUN: opt -S -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -verify-each -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
; A uniform pointer argument of an internal function, passed from a kernel,
; is promoted to inreg (SGPR) on both the definition and the call site.
-; CHECK-LABEL: define internal fastcc void @callee_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+; CHECK-LABEL: define internal fastcc void @callee_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -14,7 +20,11 @@ define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
define amdgpu_kernel void @k_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
-; CHECK: call fastcc void @callee_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_uniform(ptr %p, i32 %tid)
ret void
@@ -23,10 +33,13 @@ define amdgpu_kernel void @k_uniform(ptr %p) {
; A divergent pointer operand (derived from the workitem id) must NOT be
; promoted, because inreg would drop all but one lane's value.
-; CHECK-LABEL: define internal fastcc void @callee_divergent(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_divergent(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -34,7 +47,12 @@ define internal fastcc void @callee_divergent(ptr %p) {
define amdgpu_kernel void @k_divergent(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
-; CHECK: call fastcc void @callee_divergent(ptr %pdiv)
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @callee_divergent(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%pdiv = getelementptr float, ptr %base, i32 %tid
call fastcc void @callee_divergent(ptr %pdiv)
@@ -44,17 +62,22 @@ define amdgpu_kernel void @k_divergent(ptr %base) {
; Private pointers name lane-private storage. Even if the pointer value itself
; is uniform, the callee must not learn that the pointee is wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_private(
-; CHECK-SAME: ptr addrspace(5) %p
-; CHECK-NOT: ptr addrspace(5) inreg
define internal fastcc void @callee_private(ptr addrspace(5) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_private(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr addrspace(5) %p
ret void
}
define amdgpu_kernel void @k_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_private(
-; CHECK: call fastcc void @callee_private(ptr addrspace(5) %a)
+; CHECK-LABEL: define amdgpu_kernel void @k_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: call fastcc void @callee_private(ptr addrspace(5) [[A]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
call fastcc void @callee_private(ptr addrspace(5) %a)
ret void
@@ -62,17 +85,23 @@ define amdgpu_kernel void @k_private() {
; A flat pointer derived from private memory carries the same risk.
-; CHECK-LABEL: define internal fastcc void @callee_flat_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_flat_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_flat_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr %p
ret void
}
define amdgpu_kernel void @k_flat_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_flat_private(
-; CHECK: call fastcc void @callee_flat_private(ptr %f)
+; CHECK-LABEL: define amdgpu_kernel void @k_flat_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: [[F:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
+; CHECK-NEXT: call fastcc void @callee_flat_private(ptr [[F]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
%f = addrspacecast ptr addrspace(5) %a to ptr
call fastcc void @callee_flat_private(ptr %f)
@@ -82,10 +111,12 @@ define amdgpu_kernel void @k_flat_private() {
; A flat pointer that may be private on one path of a phi/select must not be
; promoted, even though the other path is a benign global pointer.
-; CHECK-LABEL: define internal fastcc void @callee_phi_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_phi_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_phi_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr %p
ret void
}
@@ -94,7 +125,14 @@ define internal fastcc void @callee_phi_private(ptr %p) {
define amdgpu_kernel void @k_phi_private(i1 %c) {
; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
-; CHECK: call fastcc void @callee_phi_private(ptr %sel)
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
+; CHECK-NEXT: [[FA:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
+; CHECK-NEXT: [[FG:%.*]] = addrspacecast ptr addrspace(1) @gvar to ptr
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr [[FA]], ptr [[FG]]
+; CHECK-NEXT: call fastcc void @callee_phi_private(ptr [[SEL]])
+; CHECK-NEXT: ret void
+;
%a = alloca i32, addrspace(5)
%fa = addrspacecast ptr addrspace(5) %a to ptr
%fg = addrspacecast ptr addrspace(1) @gvar to ptr
@@ -106,17 +144,34 @@ define amdgpu_kernel void @k_phi_private(i1 %c) {
; A private pointer reached through a long getelementptr chain stays in the
; private address space, so the address-space check still blocks it.
-; CHECK-LABEL: define internal fastcc void @callee_deep_private(
-; CHECK-SAME: ptr addrspace(5) %p
-; CHECK-NOT: ptr addrspace(5) inreg
define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_deep_private(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 1, ptr addrspace(5) %p
ret void
}
define amdgpu_kernel void @k_deep_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_deep_private(
-; CHECK: call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+; CHECK-LABEL: define amdgpu_kernel void @k_deep_private() {
+; CHECK-NEXT: [[A:%.*]] = alloca [64 x i32], align 4, addrspace(5)
+; CHECK-NEXT: [[G1:%.*]] = getelementptr i32, ptr addrspace(5) [[A]], i32 1
+; CHECK-NEXT: [[G2:%.*]] = getelementptr i32, ptr addrspace(5) [[G1]], i32 1
+; CHECK-NEXT: [[G3:%.*]] = getelementptr i32, ptr addrspace(5) [[G2]], i32 1
+; CHECK-NEXT: [[G4:%.*]] = getelementptr i32, ptr addrspace(5) [[G3]], i32 1
+; CHECK-NEXT: [[G5:%.*]] = getelementptr i32, ptr addrspace(5) [[G4]], i32 1
+; CHECK-NEXT: [[G6:%.*]] = getelementptr i32, ptr addrspace(5) [[G5]], i32 1
+; CHECK-NEXT: [[G7:%.*]] = getelementptr i32, ptr addrspace(5) [[G6]], i32 1
+; CHECK-NEXT: [[G8:%.*]] = getelementptr i32, ptr addrspace(5) [[G7]], i32 1
+; CHECK-NEXT: [[G9:%.*]] = getelementptr i32, ptr addrspace(5) [[G8]], i32 1
+; CHECK-NEXT: [[G10:%.*]] = getelementptr i32, ptr addrspace(5) [[G9]], i32 1
+; CHECK-NEXT: [[G11:%.*]] = getelementptr i32, ptr addrspace(5) [[G10]], i32 1
+; CHECK-NEXT: [[G12:%.*]] = getelementptr i32, ptr addrspace(5) [[G11]], i32 1
+; CHECK-NEXT: call fastcc void @callee_deep_private(ptr addrspace(5) [[G12]])
+; CHECK-NEXT: ret void
+;
%a = alloca [64 x i32], addrspace(5)
%g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
%g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
@@ -138,10 +193,13 @@ define amdgpu_kernel void @k_deep_private() {
; callee reinterprets it as private (scratch), because that extracts a
; lane-relative offset that is not wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_casts_private(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_casts_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_casts_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
%q = addrspacecast ptr %p to ptr addrspace(5)
store i32 1, ptr addrspace(5) %q
ret void
@@ -149,7 +207,10 @@ define internal fastcc void @callee_casts_private(ptr %p) {
define amdgpu_kernel void @k_casts_private(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
-; CHECK: call fastcc void @callee_casts_private(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_casts_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_casts_private(ptr %p)
ret void
}
@@ -157,9 +218,13 @@ define amdgpu_kernel void @k_casts_private(ptr %p) {
; Positive control: a flat load/store through the argument is the common, safe
; use and must still be promoted.
-; CHECK-LABEL: define internal fastcc void @callee_flat_load(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @callee_flat_load(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_flat_load(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -167,35 +232,45 @@ define internal fastcc void @callee_flat_load(ptr %p) {
define amdgpu_kernel void @k_flat_load(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
-; CHECK: call fastcc void @callee_flat_load(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_flat_load(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_flat_load(ptr %p)
ret void
}
; Multiple uniform pointer arguments are all promoted opportunistically.
-; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
- ptr %p4) {
+; CHECK-LABEL: define internal fastcc void @callee_budget(
+; CHECK-SAME: ptr inreg [[P0:%.*]], ptr inreg [[P1:%.*]], ptr inreg [[P2:%.*]], ptr inreg [[P3:%.*]], ptr inreg [[P4:%.*]]) {
+; CHECK-NEXT: ret void
+;
+ ptr %p4) {
ret void
}
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
- ptr %p4) {
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK: call fastcc void @callee_budget(ptr inreg %p0, ptr inreg %p1, ptr inreg %p2, ptr inreg %p3, ptr inreg %p4)
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_budget(ptr inreg [[P0]], ptr inreg [[P1]], ptr inreg [[P2]], ptr inreg [[P3]], ptr inreg [[P4]])
+; CHECK-NEXT: ret void
+;
+ ptr %p4) {
call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
ret void
}
-; TTI cross-check: a flat load is NeverUniform in GCNTTI even when the address
-; is wave-uniform, so the operand must not be promoted.
+; Trivial check: a loaded pointer is not AlwaysUniform, so the operand must not
+; be promoted even when the load address is wave-uniform.
-; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_tti_flatload(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
@@ -203,8 +278,12 @@ define internal fastcc void @callee_tti_flatload(ptr %p) {
@gptr = addrspace(1) global ptr null
define amdgpu_kernel void @k_tti_flatload() {
-; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload(
-; CHECK: call fastcc void @callee_tti_flatload(ptr %p)
+; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload() {
+; CHECK-NEXT: [[FLATG:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
+; CHECK-NEXT: [[P:%.*]] = load ptr, ptr [[FLATG]], align 8
+; CHECK-NEXT: call fastcc void @callee_tti_flatload(ptr [[P]])
+; CHECK-NEXT: ret void
+;
%flatg = addrspacecast ptr addrspace(1) @gptr to ptr
%p = load ptr, ptr %flatg
call fastcc void @callee_tti_flatload(ptr %p)
@@ -214,16 +293,24 @@ define amdgpu_kernel void @k_tti_flatload() {
; External linkage: all call sites are not necessarily visible, so the ABI
; must not be changed.
-; CHECK-LABEL: define fastcc void @callee_external(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define fastcc void @callee_external(ptr %p) {
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
}
define amdgpu_kernel void @k_external(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_external(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_external(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_external(ptr %p)
ret void
}
@@ -231,10 +318,13 @@ define amdgpu_kernel void @k_external(ptr %p) {
; Address-taken internal function: an indirect call we cannot see could pass a
; divergent value, so do not promote.
-; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_addrtaken(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -243,6 +333,12 @@ define internal fastcc void @callee_addrtaken(ptr %p) {
@fnptr = global ptr null
define amdgpu_kernel void @k_addrtaken(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_addrtaken(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @callee_addrtaken, ptr @fnptr, align 8
+; CHECK-NEXT: call fastcc void @callee_addrtaken(ptr [[P]])
+; CHECK-NEXT: ret void
+;
store ptr @callee_addrtaken, ptr @fnptr
call fastcc void @callee_addrtaken(ptr %p)
ret void
@@ -252,77 +348,99 @@ define amdgpu_kernel void @k_addrtaken(ptr %p) {
%struct.Foo = type { i32 }
-; CHECK-LABEL: define internal fastcc void @callee_byref(
-; CHECK-SAME: ptr byref(%struct.Foo) %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
+; CHECK-LABEL: define internal fastcc void @callee_byref(
+; CHECK-SAME: ptr byref([[STRUCT_FOO:%.*]]) [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_byref(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_byref(
-; CHECK: call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_byref(ptr byref([[STRUCT_FOO:%.*]]) [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_byref(ptr byref(%struct.Foo) %p)
ret void
}
-; CHECK-LABEL: define internal fastcc ptr @callee_returned(
-; CHECK-SAME: ptr returned %p
-; CHECK-NOT: ptr inreg
define internal fastcc ptr @callee_returned(ptr returned %p) {
+; CHECK-LABEL: define internal fastcc ptr @callee_returned(
+; CHECK-SAME: ptr returned [[P:%.*]]) {
+; CHECK-NEXT: ret ptr [[P]]
+;
ret ptr %p
}
define amdgpu_kernel void @k_returned(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_returned(
-; CHECK: call fastcc ptr @callee_returned(ptr returned %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call fastcc ptr @callee_returned(ptr returned [[P]])
+; CHECK-NEXT: store ptr [[R]], ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%r = call fastcc ptr @callee_returned(ptr returned %p)
store ptr %r, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
-; CHECK-SAME: ptr swiftasync %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
+; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
+; CHECK-SAME: ptr swiftasync [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_swiftasync(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
-; CHECK: call fastcc void @callee_swiftasync(ptr swiftasync %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_swiftasync(ptr swiftasync [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_swiftasync(ptr swiftasync %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_hidden(
-; CHECK-SAME: ptr "amdgpu-hidden-argument" %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
+; CHECK-LABEL: define internal fastcc void @callee_hidden(
+; CHECK-SAME: ptr "amdgpu-hidden-argument" [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
define amdgpu_kernel void @k_hidden(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
-; CHECK: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p)
ret void
}
; Non-pointer uniform arguments are out of scope for this pass.
-; CHECK-LABEL: define internal fastcc void @callee_scalar(
-; CHECK-SAME: i32 %n
-; CHECK-NOT: i32 inreg
define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 [[N:%.*]], ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store i32 %n, ptr %p
ret void
}
define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_scalar(i32 [[N]], ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
; The pointer is still promoted, the scalar is not.
-; CHECK: call fastcc void @callee_scalar(i32 %n, ptr inreg %p)
call fastcc void @callee_scalar(i32 %n, ptr %p)
ret void
}
@@ -330,10 +448,13 @@ define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
; Mixed call sites: one uniform, one divergent. A single divergent operand must
; block promotion, since the definition is shared by all callers.
-; CHECK-LABEL: define internal fastcc void @callee_mixed(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_mixed(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_mixed(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%v = load float, ptr %p
store float %v, ptr %p
ret void
@@ -341,14 +462,22 @@ define internal fastcc void @callee_mixed(ptr %p) {
define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
-; CHECK: call fastcc void @callee_mixed(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_mixed(ptr %p)
ret void
}
define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
-; CHECK: call fastcc void @callee_mixed(ptr %pdiv)
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%pdiv = getelementptr float, ptr %base, i32 %tid
call fastcc void @callee_mixed(ptr %pdiv)
@@ -358,32 +487,42 @@ define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
; Multi-hop chain: uniformity propagates from the kernel through each internal
; function to a fixpoint, so every hop's closure pointer is promoted.
-; CHECK-LABEL: define internal fastcc void @chain_leaf(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @chain_mid(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_mid(ptr %p) {
-; CHECK: call fastcc void @chain_leaf(ptr inreg %p)
+; CHECK-LABEL: define internal fastcc void @chain_mid(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_leaf(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_leaf(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @chain_top(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @chain_top(ptr %p) {
-; CHECK: call fastcc void @chain_mid(ptr inreg %p)
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_mid(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_mid(ptr %p)
ret void
}
define amdgpu_kernel void @k_chain(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_chain(
-; CHECK: call fastcc void @chain_top(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_top(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @chain_top(ptr %p)
ret void
}
@@ -391,58 +530,83 @@ define amdgpu_kernel void @k_chain(ptr %p) {
; Fixpoint must converge regardless of the order the functions appear in the
; module (callees defined before/after their callers) and across a diamond.
-; CHECK-LABEL: define internal fastcc void @scram_a(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_a(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_a(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_b(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_b(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @scram_c(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_c(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_c(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @scram_b(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @scram_b(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @scram_b(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_c(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_c(ptr %p)
ret void
}
define amdgpu_kernel void @k_scram(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scram(
-; CHECK: call fastcc void @scram_a(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_a(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @scram_a(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_bot(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_bot(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_bot(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_l(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_l(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_l(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_bot(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_r(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_r(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_r(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_bot(ptr %p)
ret void
}
-; CHECK-LABEL: define internal fastcc void @diam_top(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @diam_top(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @diam_top(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_l(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @diam_r(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_l(ptr %p)
call fastcc void @diam_r(ptr %p)
ret void
@@ -450,7 +614,10 @@ define internal fastcc void @diam_top(ptr %p) {
define amdgpu_kernel void @k_diam(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_diam(
-; CHECK: call fastcc void @diam_top(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_top(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @diam_top(ptr %p)
ret void
}
@@ -458,16 +625,22 @@ define amdgpu_kernel void @k_diam(ptr %p) {
; An argument that is already inreg must be left untouched (no double attribute,
; no crash).
-; CHECK-LABEL: define internal fastcc void @callee_already(
-; CHECK-SAME: ptr inreg %p
define internal fastcc void @callee_already(ptr inreg %p) {
+; CHECK-LABEL: define internal fastcc void @callee_already(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_already(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_already(
-; CHECK: call fastcc void @callee_already(ptr inreg %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_already(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_already(ptr inreg %p)
ret void
}
@@ -475,17 +648,28 @@ define amdgpu_kernel void @k_already(ptr %p) {
; Invoke call sites are not audited for inreg ABI consistency under exceptional
; control flow, so promotion is skipped.
-; CHECK-LABEL: define internal fastcc void @callee_invoke(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_invoke(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_invoke(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
-; CHECK: invoke fastcc void @callee_invoke(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) personality ptr null {
+; CHECK-NEXT: invoke fastcc void @callee_invoke(ptr [[P]])
+; CHECK-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD:.*]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[LPAD]]:
+; CHECK-NEXT: [[TOK:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: ret void
+;
invoke fastcc void @callee_invoke(ptr %p) to label %cont unwind label %lpad
cont:
@@ -493,25 +677,30 @@ cont:
lpad:
%tok = landingpad { ptr, i32 }
- cleanup
+ cleanup
ret void
}
; Indirect call through a bitcast of the function pointer: the callee is not a
; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
-; CHECK-LABEL: define internal fastcc void @callee_bitcast(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_bitcast(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_bitcast(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_bitcast(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK: call void {{.*}}(ptr {{.*}}%p)
-; CHECK-NOT: inreg
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[FN:%.*]] = bitcast ptr @callee_bitcast to ptr
+; CHECK-NEXT: call void [[FN]](ptr [[P]])
+; CHECK-NEXT: ret void
+;
%fn = bitcast ptr @callee_bitcast to ptr
call void %fn(ptr %p)
ret void
@@ -520,10 +709,12 @@ define amdgpu_kernel void @k_bitcast(ptr %p) {
; A function stored into a global (non-call use) is not eligible even if there
; is also a direct call the pass can see.
-; CHECK-LABEL: define internal fastcc void @callee_stored(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_stored(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_stored(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
store float 0.000000e+00, ptr %p
ret void
}
@@ -532,7 +723,11 @@ define internal fastcc void @callee_stored(ptr %p) {
define amdgpu_kernel void @k_stored(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_stored(
-; CHECK: call fastcc void @callee_stored(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @callee_stored, ptr @fn_slot, align 8
+; CHECK-NEXT: call fastcc void @callee_stored(ptr [[P]])
+; CHECK-NEXT: ret void
+;
store ptr @callee_stored, ptr @fn_slot
call fastcc void @callee_stored(ptr %p)
ret void
@@ -541,20 +736,29 @@ define amdgpu_kernel void @k_stored(ptr %p) {
; musttail forwarding of an argument blocks inreg promotion for that argument.
define internal fastcc void @tail_target(ptr %q) {
+; CHECK-LABEL: define internal fastcc void @tail_target(
+; CHECK-SAME: ptr [[Q:%.*]]) {
+; CHECK-NEXT: ret void
+;
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
-; CHECK-SAME: ptr %p
-; CHECK-NOT: ptr inreg
define internal fastcc void @callee_musttail_forward(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: musttail call fastcc void @tail_target(ptr [[P]])
+; CHECK-NEXT: ret void
+;
musttail call fastcc void @tail_target(ptr %p)
ret void
}
define amdgpu_kernel void @k_musttail_forward(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
-; CHECK: call fastcc void @callee_musttail_forward(ptr %p)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_musttail_forward(ptr [[P]])
+; CHECK-NEXT: ret void
+;
call fastcc void @callee_musttail_forward(ptr %p)
ret void
}
@@ -562,9 +766,14 @@ define amdgpu_kernel void @k_musttail_forward(ptr %p) {
; inlinehint and alwaysinline callees behave the same for inreg promotion when
; the callee remains an out-of-line call (the motivating Kokkos case).
-; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -573,15 +782,24 @@ define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
-; CHECK: call fastcc void @callee_inlinehint_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_inlinehint_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
ret void
}
-; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
-; CHECK-SAME: ptr inreg {{.*}}%p
define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
+; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
@@ -590,13 +808,419 @@ define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
-; CHECK: call fastcc void @callee_alwaysinline_uniform(ptr inreg %p, i32 %tid)
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_alwaysinline_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: ret void
+;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
ret void
}
+; readfirstlane produces an always-uniform value (per TargetTransformInfo), so a
+; pointer laundered through it is promoted even though its input is divergent.
+
+define internal fastcc void @callee_readfirstlane(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_readfirstlane(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_readfirstlane(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_readfirstlane(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @callee_readfirstlane(ptr inreg [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @callee_readfirstlane(ptr %puni)
+ ret void
+}
+
+; A single callee reached from two kernels through different uniform sources (a
+; kernel SGPR argument and a readfirstlane result) is still promoted.
+
+define internal fastcc void @multi_src(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @multi_src(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_sgpr(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_sgpr(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @multi_src(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_readfirstlane(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_readfirstlane(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @multi_src(ptr %puni)
+ ret void
+}
+
+; Negative counterpart of the multi-source shape: one call site passes a uniform
+; (readfirstlane) operand and the other passes a divergent one. A single
+; divergent source blocks promotion of the shared definition, so no inreg appears
+; on the definition or at either call site.
+
+define internal fastcc void @multi_src_mixed(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @multi_src_mixed(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_mixed_uniform(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_uniform(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
+; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PUNI]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
+ call fastcc void @multi_src_mixed(ptr %puni)
+ ret void
+}
+
+define amdgpu_kernel void @k_multi_mixed_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_divergent(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PDIV]])
+; CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @multi_src_mixed(ptr %pdiv)
+ ret void
+}
+
+; Self-recursive callee that forwards its own pointer argument: the fixpoint must
+; terminate. Promotion is conservatively skipped because the recursive call site
+; forwards the argument back to itself (the cycle cannot be proven uniform here).
+
+define internal fastcc void @self_rec(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @self_rec(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 [[N1]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ %n1 = sub i32 %n, 1
+ call fastcc void @self_rec(ptr %p, i32 %n1)
+ br label %end
+end:
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_self_rec(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_self_rec(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 10)
+; CHECK-NEXT: ret void
+;
+ call fastcc void @self_rec(ptr %p, i32 10)
+ ret void
+}
+
+; Mutually recursive callees that forward the pointer around a cycle: likewise
+; must terminate and are conservatively left unpromoted.
+
+define internal fastcc void @mutual_a(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @mutual_a(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: call fastcc void @mutual_b(ptr [[P]], i32 [[N1]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ %n1 = sub i32 %n, 1
+ call fastcc void @mutual_b(ptr %p, i32 %n1)
+ br label %end
+end:
+ ret void
+}
+
+define internal fastcc void @mutual_b(ptr %p, i32 %n) {
+; CHECK-LABEL: define internal fastcc void @mutual_b(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
+; CHECK: [[REC]]:
+; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 [[N]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ %c = icmp ne i32 %n, 0
+ br i1 %c, label %rec, label %end
+rec:
+ call fastcc void @mutual_a(ptr %p, i32 %n)
+ br label %end
+end:
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define amdgpu_kernel void @k_mutual(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mutual(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 5)
+; CHECK-NEXT: ret void
+;
+ call fastcc void @mutual_a(ptr %p, i32 5)
+ ret void
+}
+
+; An intermediate caller that is address-taken could be reached indirectly with a
+; divergent pointer, so a leaf reached only through it must NOT be promoted even
+; though the visible direct call is uniform.
+
+ at addr_slot = global ptr null
+
+define internal fastcc void @ci_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci_leaf(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define internal fastcc void @ci_mid(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci_mid(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci_leaf(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_ci(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ci(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @ci_mid, ptr @addr_slot, align 8
+; CHECK-NEXT: call fastcc void @ci_mid(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ store ptr @ci_mid, ptr @addr_slot
+ call fastcc void @ci_mid(ptr %p)
+ ret void
+}
+
+; The same shared leaf reached through two intermediates: one address-taken, one
+; clean. The address-taken path taints the leaf (a single unprovable call site
+; blocks the shared definition), so @ci2_leaf is NOT promoted. The clean
+; intermediate @ci2_mid_clean is still promoted on its own, because its only call
+; site is the uniform kernel call and promotion is decided per callee.
+
+ at addr_slot2 = global ptr null
+
+define internal fastcc void @ci2_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_leaf(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ store float 0.000000e+00, ptr %p
+ ret void
+}
+
+define internal fastcc void @ci2_mid_at(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_mid_at(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci2_leaf(ptr %p)
+ ret void
+}
+
+define internal fastcc void @ci2_mid_clean(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @ci2_mid_clean(
+; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @ci2_leaf(ptr %p)
+ ret void
+}
+
+define amdgpu_kernel void @k_ci2(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ci2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: store ptr @ci2_mid_at, ptr @addr_slot2, align 8
+; CHECK-NEXT: call fastcc void @ci2_mid_at(ptr [[P]])
+; CHECK-NEXT: call fastcc void @ci2_mid_clean(ptr inreg [[P]])
+; CHECK-NEXT: ret void
+;
+ store ptr @ci2_mid_at, ptr @addr_slot2
+ call fastcc void @ci2_mid_at(ptr %p)
+ call fastcc void @ci2_mid_clean(ptr %p)
+ ret void
+}
+
+; A vector-of-pointers argument is not a scalar pointer and is out of scope.
+
+define internal fastcc void @callee_vecptr(<2 x ptr> %p) {
+; CHECK-LABEL: define internal fastcc void @callee_vecptr(
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define amdgpu_kernel void @k_vecptr(<2 x ptr> %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_vecptr(
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_vecptr(<2 x ptr> [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_vecptr(<2 x ptr> %p)
+ ret void
+}
+
+; A musttail call requires the enclosing function's parameter ABI attributes to
+; match the call positionally, regardless of which values are forwarded. Promoting
+; even an unrelated uniform argument (@p, which is not passed to the musttail
+; call) would break that contract, so no argument of a function containing a
+; musttail call is promoted.
+
+declare fastcc void @mt_abi_target(ptr, ptr)
+
+define internal fastcc void @callee_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define internal fastcc void @callee_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: musttail call fastcc void @mt_abi_target(ptr [[Q]], ptr [[Q]])
+; CHECK-NEXT: ret void
+;
+ musttail call fastcc void @mt_abi_target(ptr %q, ptr %q)
+ ret void
+}
+
+define amdgpu_kernel void @k_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_musttail_abi(ptr [[P]], ptr [[Q]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_musttail_abi(ptr %p, ptr %q)
+ ret void
+}
+
+; A freeze between the argument and a private addrspacecast must not hide the
+; private reinterpretation from the callee-side guard.
+
+define internal fastcc void @callee_freeze_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_freeze_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = freeze ptr [[P]]
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[F]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
+ %f = freeze ptr %p
+ %q = addrspacecast ptr %f to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_freeze_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_freeze_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_freeze_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_freeze_private(ptr %p)
+ ret void
+}
+
+; ptrmask likewise forwards the pointer; a private cast behind it must still be
+; detected by the callee-side guard.
+
+define internal fastcc void @callee_ptrmask_private(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_ptrmask_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[M:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -16)
+; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[M]] to ptr addrspace(5)
+; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+; CHECK-NEXT: ret void
+;
+ %m = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -16)
+ %q = addrspacecast ptr %m to ptr addrspace(5)
+ store i32 1, ptr addrspace(5) %q
+ ret void
+}
+
+define amdgpu_kernel void @k_ptrmask_private(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ptrmask_private(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_ptrmask_private(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_ptrmask_private(ptr %p)
+ ret void
+}
+
attributes #0 = { inlinehint }
attributes #1 = { alwaysinline }
declare i32 @llvm.amdgcn.workitem.id.x()
+declare ptr @llvm.amdgcn.readfirstlane.p0(ptr)
+declare ptr @llvm.ptrmask.p0.i64(ptr, i64)
>From ec75bda68fbff73fc7c39f2d84cbdae6fb489103 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:45:15 -0500
Subject: [PATCH 07/10] Skip optnone in promote-uniform-args; Query
GCNTTIImpl::isAlwaysUniform via TTIs.
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 42 ++++++++++++-------
.../CodeGen/AMDGPU/promote-uniform-args.ll | 39 +++++++++++++++++
2 files changed, 67 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 59d77698aae30..f802b83d85eb1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -7,11 +7,13 @@
//===----------------------------------------------------------------------===//
//
// Promote provably-uniform pointer arguments of internal callees to \c inreg
-// (SGPR passing) on the definition and at every direct call site. Uniformity is
-// established conservatively: an operand is uniform if it is an always-uniform
-// instruction (per TargetTransformInfo) or an argument that every visible
-// caller forwards from another uniform value. A full \c UniformityInfo-based
-// check is a planned follow-up.
+// (SGPR passing) on the definition and at every direct call site. Pointers are
+// the primary target: non-entry pointer args default to VGPR unless marked
+// \c inreg, while many scalars already use the SGPR path. Scalar promotion is
+// a planned follow-up. Uniformity is established conservatively via
+// \c GCNTTIImpl::isAlwaysUniform (queried through TTI) plus recursive
+// propagation through visible caller chains. A full \c UniformityInfo check is
+// a planned follow-up.
//
//===----------------------------------------------------------------------===//
@@ -147,6 +149,13 @@ static bool calleeCastsArgToPrivate(const Argument &A) {
return false;
}
+static bool isAlwaysUniformValue(const TargetTransformInfo &TTI,
+ const Value *V) {
+ if (isa<Constant>(V))
+ return true;
+ return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
+}
+
static bool isTriviallyUniformArg(
const Argument *Arg,
function_ref<TargetTransformInfo &(const Function &)> GetTTI,
@@ -159,10 +168,8 @@ static bool isTriviallyUniformOperand(
if (const auto *Arg = dyn_cast<Argument>(V))
return isTriviallyUniformArg(Arg, GetTTI, Visited);
- if (const auto *I = dyn_cast<Instruction>(V)) {
- const TargetTransformInfo &TTI = GetTTI(*I->getFunction());
- return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
- }
+ if (const auto *I = dyn_cast<Instruction>(V))
+ return isAlwaysUniformValue(GetTTI(*I->getFunction()), V);
return false;
}
@@ -218,6 +225,8 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
while (RoundChanged) {
RoundChanged = false;
for (Function &F : M) {
+ if (F.hasOptNone())
+ continue;
if (!isEligibleInRegUniformCallee(F))
continue;
@@ -230,11 +239,16 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
// isEligibleInRegUniformCallee guarantees every user is a direct,
// non-musttail, non-invoke call to F, so we can just gather them.
SmallVector<CallBase *, 8> Calls;
- for (User *U : F.users())
- if (auto *CB = dyn_cast<CallBase>(U);
- CB && CB->getCalledFunction() == &F)
- Calls.push_back(CB);
- if (Calls.empty())
+ bool HasOptNoneCaller = false;
+ for (User *U : F.users()) {
+ auto *CB = dyn_cast<CallBase>(U);
+ if (!CB || CB->getCalledFunction() != &F)
+ continue;
+ if (CB->getFunction()->hasOptNone())
+ HasOptNoneCaller = true;
+ Calls.push_back(CB);
+ }
+ if (HasOptNoneCaller || Calls.empty())
continue;
bool FuncChanged = false;
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 64bff2fc40644..786cdefa5218b 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1218,8 +1218,47 @@ define amdgpu_kernel void @k_ptrmask_private(ptr %p) {
ret void
}
+; optnone on the caller: ABI attributes must not be changed.
+define internal fastcc void @callee_optnone_caller(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_optnone_caller(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define amdgpu_kernel void @k_optnone_caller(ptr %p) #2 {
+; CHECK-LABEL: define amdgpu_kernel void @k_optnone_caller(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT: call fastcc void @callee_optnone_caller(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_optnone_caller(ptr %p)
+ ret void
+}
+
+; optnone on the callee itself.
+define internal fastcc void @callee_optnone_callee(ptr %p) #2 {
+; CHECK-LABEL: define internal fastcc void @callee_optnone_callee(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define amdgpu_kernel void @k_optnone_callee(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_optnone_callee(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @callee_optnone_callee(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_optnone_callee(ptr %p)
+ ret void
+}
+
attributes #0 = { inlinehint }
attributes #1 = { alwaysinline }
+attributes #2 = { noinline optnone }
declare i32 @llvm.amdgcn.workitem.id.x()
declare ptr @llvm.amdgcn.readfirstlane.p0(ptr)
>From 7ea7fcdc363870d4a0cb7bc872df7cea8306fbd6 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 25 Aug 2026 10:36:16 -0500
Subject: [PATCH 08/10] further simplify trivial uniformity checks
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 258 +++---------------
.../CodeGen/AMDGPU/promote-uniform-args.ll | 76 +++---
2 files changed, 82 insertions(+), 252 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index f802b83d85eb1..43560ece260a4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -6,28 +6,21 @@
//
//===----------------------------------------------------------------------===//
//
-// Promote provably-uniform pointer arguments of internal callees to \c inreg
-// (SGPR passing) on the definition and at every direct call site. Pointers are
-// the primary target: non-entry pointer args default to VGPR unless marked
-// \c inreg, while many scalars already use the SGPR path. Scalar promotion is
-// a planned follow-up. Uniformity is established conservatively via
-// \c GCNTTIImpl::isAlwaysUniform (queried through TTI) plus recursive
-// propagation through visible caller chains. A full \c UniformityInfo check is
-// a planned follow-up.
+// Promote pointer arguments of internal callees to \c inreg when every visible
+// call-site operand is trivially uniform per \c GCNTTIImpl::isAlwaysUniform
+// (queried through TTI). Arg-chain propagation, private guards, and full
+// \c UniformityInfo are planned follow-ups; see
+// \c AMDGPUPromoteUniformArgs.cpp.advanced for a more complete prototype.
//
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
-#include "Utils/AMDGPUBaseInfo.h"
-#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
@@ -57,7 +50,7 @@ static bool canPromoteArgToInReg(const Argument &A) {
}
static bool isEligibleInRegUniformCallee(const Function &F) {
- if (F.isDeclaration() || F.isVarArg())
+ if (F.isDeclaration() || F.isVarArg() || F.hasOptNone())
return false;
if (!F.hasLocalLinkage() || F.hasAddressTaken())
return false;
@@ -74,79 +67,10 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
return false;
if (CB->isMustTailCall() || isa<InvokeInst>(CB))
return false;
+ if (CB->getFunction()->hasOptNone())
+ return false;
}
- if (F.user_empty())
- return false;
- return true;
-}
-
-// musttail requires the enclosing function's parameter ABI (including inreg) to
-// match positionally, so skip promotion for any function containing one.
-static bool hasMustTailCall(const Function &F) {
- for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (CB->isMustTailCall())
- return true;
- return false;
-}
-
-static bool mayBePrivateDerivedPointer(const Value *V) {
- assert(V->getType()->isPointerTy());
- if (V->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
-
- SmallVector<const Value *, 8> Objects;
- getUnderlyingObjects(V, Objects);
- for (const Value *Obj : Objects) {
- if (isa<AllocaInst>(Obj))
- return true;
- if (Obj->getType()->isPointerTy() &&
- Obj->getType()->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- }
- return false;
-}
-
-static bool calleeCastsArgToPrivate(const Argument &A) {
- SmallVector<const Value *, 16> Worklist;
- SmallPtrSet<const Value *, 16> Visited;
- Worklist.push_back(&A);
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- for (const User *U : V->users()) {
- if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(U)) {
- if (ASC->getDestAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(ASC);
- continue;
- }
- if (const auto *II = dyn_cast<IntrinsicInst>(U)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::amdgcn_addrspacecast_nonnull:
- if (II->getType()->getPointerAddressSpace() ==
- AMDGPUAS::PRIVATE_ADDRESS)
- return true;
- Worklist.push_back(II);
- break;
- case Intrinsic::ptrmask:
- case Intrinsic::launder_invariant_group:
- case Intrinsic::strip_invariant_group:
- Worklist.push_back(II);
- break;
- default:
- break;
- }
- continue;
- }
- if (isa<GetElementPtrInst, BitCastInst, PHINode, SelectInst, FreezeInst>(
- U))
- Worklist.push_back(U);
- }
- }
- return false;
+ return !F.user_empty();
}
static bool isAlwaysUniformValue(const TargetTransformInfo &TTI,
@@ -156,147 +80,53 @@ static bool isAlwaysUniformValue(const TargetTransformInfo &TTI,
return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
}
-static bool isTriviallyUniformArg(
- const Argument *Arg,
- function_ref<TargetTransformInfo &(const Function &)> GetTTI,
- SmallPtrSetImpl<const Argument *> &Visited);
-
-static bool isTriviallyUniformOperand(
- const Value *V,
- function_ref<TargetTransformInfo &(const Function &)> GetTTI,
- SmallPtrSetImpl<const Argument *> &Visited) {
- if (const auto *Arg = dyn_cast<Argument>(V))
- return isTriviallyUniformArg(Arg, GetTTI, Visited);
-
- if (const auto *I = dyn_cast<Instruction>(V))
- return isAlwaysUniformValue(GetTTI(*I->getFunction()), V);
-
- return false;
-}
-
-static bool isTriviallyUniformArg(
- const Argument *Arg,
- function_ref<TargetTransformInfo &(const Function &)> GetTTI,
- SmallPtrSetImpl<const Argument *> &Visited) {
- if (Arg->hasInRegAttr())
- return true;
-
- const Function *F = Arg->getParent();
- if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
- return AMDGPU::isArgPassedInSGPR(Arg);
-
- // Only direct-call-only internal callees have fully visible callers; an
- // indirect or address-taken caller could pass a divergent value we never see.
- if (!isEligibleInRegUniformCallee(*F))
- return false;
-
- // Path-based cycle guard: a (mutually) recursive chain that forwards the
- // argument back to itself cannot be proven here and must terminate. The
- // argument is removed on the way out so a diamond-shaped call graph can still
- // reach the same argument through an independent path.
- if (!Visited.insert(Arg).second)
- return false;
-
- bool HasCallSite = false;
- bool AllUniform = true;
- for (const User *U : F->users()) {
- const auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != F)
- continue;
- HasCallSite = true;
- if (!isTriviallyUniformOperand(CB->getArgOperand(Arg->getArgNo()), GetTTI,
- Visited)) {
- AllUniform = false;
- break;
- }
- }
- Visited.erase(Arg);
- return HasCallSite && AllUniform;
-}
-
static bool promoteUniformPointerArgsToInReg(Module &M,
ModuleAnalysisManager &AM) {
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- auto GetTTI = [&FAM](const Function &F) -> TargetTransformInfo & {
- return FAM.getResult<TargetIRAnalysis>(const_cast<Function &>(F));
- };
bool Changed = false;
- bool RoundChanged = true;
- while (RoundChanged) {
- RoundChanged = false;
- for (Function &F : M) {
- if (F.hasOptNone())
- continue;
- if (!isEligibleInRegUniformCallee(F))
- continue;
-
- // Promoting any argument changes F's ABI attributes, which a musttail
- // call in F's body would then no longer match positionally. Skip
- // conservatively.
- if (hasMustTailCall(F))
- continue;
- // isEligibleInRegUniformCallee guarantees every user is a direct,
- // non-musttail, non-invoke call to F, so we can just gather them.
- SmallVector<CallBase *, 8> Calls;
- bool HasOptNoneCaller = false;
- for (User *U : F.users()) {
- auto *CB = dyn_cast<CallBase>(U);
- if (!CB || CB->getCalledFunction() != &F)
- continue;
- if (CB->getFunction()->hasOptNone())
- HasOptNoneCaller = true;
- Calls.push_back(CB);
- }
- if (HasOptNoneCaller || Calls.empty())
- continue;
+ for (Function &F : M) {
+ if (!isEligibleInRegUniformCallee(F))
+ continue;
- bool FuncChanged = false;
- for (Argument &A : F.args()) {
- if (!canPromoteArgToInReg(A))
- continue;
- if (calleeCastsArgToPrivate(A))
- continue;
+ SmallVector<CallBase *, 8> Calls;
+ for (User *U : F.users()) {
+ auto *CB = cast<CallBase>(U);
+ Calls.push_back(CB);
+ }
- bool AllUniform = true;
- for (CallBase *CB : Calls) {
- Value *ArgOp = CB->getArgOperand(A.getArgNo());
- if (mayBePrivateDerivedPointer(ArgOp)) {
- AllUniform = false;
- break;
- }
+ bool FuncChanged = false;
+ for (Argument &A : F.args()) {
+ if (!canPromoteArgToInReg(A))
+ continue;
- SmallPtrSet<const Argument *, 4> VisitedArgs;
- if (!isTriviallyUniformOperand(ArgOp, GetTTI, VisitedArgs)) {
- AllUniform = false;
- break;
- }
+ bool AllUniform = true;
+ for (CallBase *CB : Calls) {
+ const TargetTransformInfo &TTI =
+ FAM.getResult<TargetIRAnalysis>(*CB->getFunction());
+ if (!isAlwaysUniformValue(TTI, CB->getArgOperand(A.getArgNo()))) {
+ AllUniform = false;
+ break;
}
- if (!AllUniform)
- continue;
-
- A.addAttr(Attribute::InReg);
- for (CallBase *CB : Calls)
- CB->addParamAttr(A.getArgNo(), Attribute::InReg);
- ++NumPromotedInRegArgs;
- FuncChanged = Changed = RoundChanged = true;
}
+ if (!AllUniform)
+ continue;
- if (FuncChanged) {
- ++NumPromotedInRegFuncs;
- PreservedAnalyses FuncPA;
- // Attribute-only change; the CFG is unchanged.
- FuncPA.preserveSet<CFGAnalyses>();
- FAM.invalidate(F, FuncPA);
- SmallPtrSet<Function *, 8> InvalidatedCallers;
- for (CallBase *CB : Calls) {
- Function *Caller = CB->getFunction();
- if (Caller != &F && InvalidatedCallers.insert(Caller).second)
- FAM.invalidate(*Caller, FuncPA);
- }
- }
+ A.addAttr(Attribute::InReg);
+ for (CallBase *CB : Calls)
+ CB->addParamAttr(A.getArgNo(), Attribute::InReg);
+ ++NumPromotedInRegArgs;
+ FuncChanged = Changed = true;
+ }
+
+ if (FuncChanged) {
+ ++NumPromotedInRegFuncs;
+ PreservedAnalyses FuncPA;
+ FuncPA.preserveSet<CFGAnalyses>();
+ FAM.invalidate(F, FuncPA);
}
}
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 786cdefa5218b..39a884f14f479 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -6,7 +6,7 @@
define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
; CHECK-LABEL: define internal fastcc void @callee_uniform(
-; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
@@ -22,7 +22,7 @@ define amdgpu_kernel void @k_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: call fastcc void @callee_uniform(ptr [[P]], i32 [[TID]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -220,7 +220,7 @@ define amdgpu_kernel void @k_casts_private(ptr %p) {
define internal fastcc void @callee_flat_load(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_flat_load(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
; CHECK-NEXT: ret void
@@ -233,7 +233,7 @@ define internal fastcc void @callee_flat_load(ptr %p) {
define amdgpu_kernel void @k_flat_load(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_flat_load(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @callee_flat_load(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @callee_flat_load(ptr %p)
@@ -244,7 +244,7 @@ define amdgpu_kernel void @k_flat_load(ptr %p) {
define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr inreg [[P0:%.*]], ptr inreg [[P1:%.*]], ptr inreg [[P2:%.*]], ptr inreg [[P3:%.*]], ptr inreg [[P4:%.*]]) {
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
; CHECK-NEXT: ret void
;
ptr %p4) {
@@ -254,7 +254,7 @@ define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
; CHECK-LABEL: define amdgpu_kernel void @k_budget(
; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_budget(ptr inreg [[P0]], ptr inreg [[P1]], ptr inreg [[P2]], ptr inreg [[P3]], ptr inreg [[P4]])
+; CHECK-NEXT: call fastcc void @callee_budget(ptr [[P0]], ptr [[P1]], ptr [[P2]], ptr [[P3]], ptr [[P4]])
; CHECK-NEXT: ret void
;
ptr %p4) {
@@ -426,7 +426,7 @@ define amdgpu_kernel void @k_hidden(ptr %p) {
define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_scalar(
-; CHECK-SAME: i32 [[N:%.*]], ptr inreg [[P:%.*]]) {
+; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -437,7 +437,7 @@ define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_scalar(i32 [[N]], ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @callee_scalar(i32 [[N]], ptr [[P]])
; CHECK-NEXT: ret void
;
; The pointer is still promoted, the scalar is not.
@@ -489,7 +489,7 @@ define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
define internal fastcc void @chain_leaf(ptr %p) {
; CHECK-LABEL: define internal fastcc void @chain_leaf(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -499,8 +499,8 @@ define internal fastcc void @chain_leaf(ptr %p) {
define internal fastcc void @chain_mid(ptr %p) {
; CHECK-LABEL: define internal fastcc void @chain_mid(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_leaf(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_leaf(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @chain_leaf(ptr %p)
@@ -509,8 +509,8 @@ define internal fastcc void @chain_mid(ptr %p) {
define internal fastcc void @chain_top(ptr %p) {
; CHECK-LABEL: define internal fastcc void @chain_top(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_mid(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @chain_mid(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @chain_mid(ptr %p)
@@ -520,7 +520,7 @@ define internal fastcc void @chain_top(ptr %p) {
define amdgpu_kernel void @k_chain(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_chain(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_top(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @chain_top(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @chain_top(ptr %p)
@@ -532,8 +532,8 @@ define amdgpu_kernel void @k_chain(ptr %p) {
define internal fastcc void @scram_a(ptr %p) {
; CHECK-LABEL: define internal fastcc void @scram_a(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_b(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_b(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @scram_b(ptr %p)
@@ -542,7 +542,7 @@ define internal fastcc void @scram_a(ptr %p) {
define internal fastcc void @scram_c(ptr %p) {
; CHECK-LABEL: define internal fastcc void @scram_c(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -552,8 +552,8 @@ define internal fastcc void @scram_c(ptr %p) {
define internal fastcc void @scram_b(ptr %p) {
; CHECK-LABEL: define internal fastcc void @scram_b(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_c(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @scram_c(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @scram_c(ptr %p)
@@ -563,7 +563,7 @@ define internal fastcc void @scram_b(ptr %p) {
define amdgpu_kernel void @k_scram(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_scram(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_a(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @scram_a(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @scram_a(ptr %p)
@@ -572,7 +572,7 @@ define amdgpu_kernel void @k_scram(ptr %p) {
define internal fastcc void @diam_bot(ptr %p) {
; CHECK-LABEL: define internal fastcc void @diam_bot(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -582,8 +582,8 @@ define internal fastcc void @diam_bot(ptr %p) {
define internal fastcc void @diam_l(ptr %p) {
; CHECK-LABEL: define internal fastcc void @diam_l(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @diam_bot(ptr %p)
@@ -592,8 +592,8 @@ define internal fastcc void @diam_l(ptr %p) {
define internal fastcc void @diam_r(ptr %p) {
; CHECK-LABEL: define internal fastcc void @diam_r(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_bot(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_bot(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @diam_bot(ptr %p)
@@ -602,9 +602,9 @@ define internal fastcc void @diam_r(ptr %p) {
define internal fastcc void @diam_top(ptr %p) {
; CHECK-LABEL: define internal fastcc void @diam_top(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_l(ptr inreg [[P]])
-; CHECK-NEXT: call fastcc void @diam_r(ptr inreg [[P]])
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: call fastcc void @diam_l(ptr [[P]])
+; CHECK-NEXT: call fastcc void @diam_r(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @diam_l(ptr %p)
@@ -615,7 +615,7 @@ define internal fastcc void @diam_top(ptr %p) {
define amdgpu_kernel void @k_diam(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_diam(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_top(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @diam_top(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @diam_top(ptr %p)
@@ -768,7 +768,7 @@ define amdgpu_kernel void @k_musttail_forward(ptr %p) {
define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
-; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
@@ -784,7 +784,7 @@ define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_inlinehint_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: call fastcc void @callee_inlinehint_uniform(ptr [[P]], i32 [[TID]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -794,7 +794,7 @@ define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
-; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
@@ -810,7 +810,7 @@ define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_alwaysinline_uniform(ptr inreg [[P]], i32 [[TID]])
+; CHECK-NEXT: call fastcc void @callee_alwaysinline_uniform(ptr [[P]], i32 [[TID]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -852,7 +852,7 @@ define amdgpu_kernel void @k_readfirstlane(ptr %base) {
define internal fastcc void @multi_src(ptr %p) {
; CHECK-LABEL: define internal fastcc void @multi_src(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -863,7 +863,7 @@ define internal fastcc void @multi_src(ptr %p) {
define amdgpu_kernel void @k_multi_sgpr(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_multi_sgpr(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @multi_src(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @multi_src(ptr %p)
@@ -876,7 +876,7 @@ define amdgpu_kernel void @k_multi_readfirstlane(ptr %base) {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
-; CHECK-NEXT: call fastcc void @multi_src(ptr inreg [[PUNI]])
+; CHECK-NEXT: call fastcc void @multi_src(ptr [[PUNI]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1094,7 +1094,7 @@ define internal fastcc void @ci2_mid_at(ptr %p) {
define internal fastcc void @ci2_mid_clean(ptr %p) {
; CHECK-LABEL: define internal fastcc void @ci2_mid_clean(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
; CHECK-NEXT: ret void
;
@@ -1107,7 +1107,7 @@ define amdgpu_kernel void @k_ci2(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: store ptr @ci2_mid_at, ptr @addr_slot2, align 8
; CHECK-NEXT: call fastcc void @ci2_mid_at(ptr [[P]])
-; CHECK-NEXT: call fastcc void @ci2_mid_clean(ptr inreg [[P]])
+; CHECK-NEXT: call fastcc void @ci2_mid_clean(ptr [[P]])
; CHECK-NEXT: ret void
;
store ptr @ci2_mid_at, ptr @addr_slot2
>From cf24f0750ea05ee4b92b4771976b960fb305da48 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 25 Aug 2026 14:53:48 -0500
Subject: [PATCH 09/10] fix passmixin
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 839204dbd01b5..a74f3f159a900 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -322,7 +322,7 @@ struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
};
struct AMDGPUPromoteUniformArgsPass
- : PassInfoMixin<AMDGPUPromoteUniformArgsPass> {
+ : OptionalPassInfoMixin<AMDGPUPromoteUniformArgsPass> {
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
};
>From 991efeceda445552e3f0283de053132ffaa99aa1 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Wed, 26 Aug 2026 12:50:17 -0500
Subject: [PATCH 10/10] include non-vector scalars;add tests;update signature
change and triviality check
---
.../AMDGPU/AMDGPUPromoteUniformArgs.cpp | 80 +-
.../CodeGen/AMDGPU/promote-uniform-args.ll | 1102 +++++------------
2 files changed, 347 insertions(+), 835 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
index 43560ece260a4..0783dc3349126 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteUniformArgs.cpp
@@ -6,53 +6,57 @@
//
//===----------------------------------------------------------------------===//
//
-// Promote pointer arguments of internal callees to \c inreg when every visible
-// call-site operand is trivially uniform per \c GCNTTIImpl::isAlwaysUniform
-// (queried through TTI). Arg-chain propagation, private guards, and full
-// \c UniformityInfo are planned follow-ups; see
-// \c AMDGPUPromoteUniformArgs.cpp.advanced for a more complete prototype.
+// Promote scalar and pointer arguments of internal callees to \c inreg when
+// every visible call-site operand is trivially uniform: a constant, an
+// argument passed in an SGPR, or an always-uniform intrinsic in the same
+// block as the call. Vectors are not promoted.
//
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/TargetParser/Triple.h"
using namespace llvm;
#define DEBUG_TYPE "amdgpu-promote-uniform-args"
STATISTIC(NumPromotedInRegArgs,
- "Number of uniform pointer arguments promoted to inreg");
+ "Number of uniform arguments promoted to inreg");
STATISTIC(NumPromotedInRegFuncs,
- "Number of functions with a promoted uniform pointer argument");
+ "Number of functions with a promoted uniform argument");
static cl::opt<bool> EnablePromoteUniformArgs(
"amdgpu-enable-promote-uniform-args", cl::Hidden, cl::init(true),
- cl::desc("Promote provably uniform internal pointer arguments to inreg"));
+ cl::desc("Promote provably uniform internal scalar and pointer arguments "
+ "to inreg"));
namespace {
static bool canPromoteArgToInReg(const Argument &A) {
- if (!A.getType()->isPointerTy() || A.hasInRegAttr())
+ Type *Ty = A.getType();
+ if (!(Ty->isIntOrPtrTy() || Ty->isFloatingPointTy()) || A.hasInRegAttr())
return false;
- if (A.hasPointeeInMemoryValueAttr() || A.hasNestAttr() ||
- A.hasReturnedAttr() || A.hasSwiftSelfAttr() || A.hasSwiftErrorAttr() ||
- A.hasAttribute(Attribute::SwiftAsync))
+ // inreg is mutually exclusive with byval, inalloca, preallocated, byref,
+ // sret, and nest. The first five are covered by hasPointeeInMemoryValueAttr.
+ if (A.hasPointeeInMemoryValueAttr() || A.hasNestAttr())
return false;
return !A.hasAttribute("amdgpu-hidden-argument");
}
static bool isEligibleInRegUniformCallee(const Function &F) {
- if (F.isDeclaration() || F.isVarArg() || F.hasOptNone())
+ if (F.isDeclaration() || F.isVarArg() || !F.canChangeSignature())
return false;
- if (!F.hasLocalLinkage() || F.hasAddressTaken())
+ if (!F.hasLocalLinkage())
return false;
switch (F.getCallingConv()) {
case CallingConv::C:
@@ -61,28 +65,46 @@ static bool isEligibleInRegUniformCallee(const Function &F) {
default:
return false;
}
+
+ // A musttail call requires the enclosing function's parameter ABI attributes
+ // to match the callee's positionally, so adding inreg to any parameter of F
+ // breaks the contract, not just to one forwarded to the tail call.
+ for (const BasicBlock &BB : F)
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (CB->isMustTailCall())
+ return false;
+
+ // Every use must be a direct call to F. This subsumes hasAddressTaken(),
+ // which by default ignores some uses we care about (e.g. assume-like calls),
+ // and it is what lets the transform treat each user as a call site to update.
for (const User *U : F.users()) {
const auto *CB = dyn_cast<CallBase>(U);
if (!CB || CB->getCalledFunction() != &F)
return false;
if (CB->isMustTailCall() || isa<InvokeInst>(CB))
return false;
- if (CB->getFunction()->hasOptNone())
- return false;
}
return !F.user_empty();
}
-static bool isAlwaysUniformValue(const TargetTransformInfo &TTI,
- const Value *V) {
+static bool isTriviallyUniform(const Use &U) {
+ Value *V = U.get();
if (isa<Constant>(V))
return true;
- return TTI.getValueUniformity(V) == ValueUniformity::AlwaysUniform;
+ if (const auto *A = dyn_cast<Argument>(V))
+ return AMDGPU::isArgPassedInSGPR(A);
+ if (const auto *II = dyn_cast<IntrinsicInst>(V)) {
+ if (!AMDGPU::isIntrinsicAlwaysUniform(II->getIntrinsicID()))
+ return false;
+ // If II and U are in different blocks then there is a possibility of
+ // temporal divergence.
+ return II->getParent() == cast<Instruction>(U.getUser())->getParent();
+ }
+ return false;
}
-static bool promoteUniformPointerArgsToInReg(Module &M,
- ModuleAnalysisManager &AM) {
- auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+static bool promoteUniformArgsToInReg(Module &M) {
bool Changed = false;
for (Function &F : M) {
@@ -102,9 +124,7 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
bool AllUniform = true;
for (CallBase *CB : Calls) {
- const TargetTransformInfo &TTI =
- FAM.getResult<TargetIRAnalysis>(*CB->getFunction());
- if (!isAlwaysUniformValue(TTI, CB->getArgOperand(A.getArgNo()))) {
+ if (!isTriviallyUniform(CB->getArgOperandUse(A.getArgNo()))) {
AllUniform = false;
break;
}
@@ -119,12 +139,8 @@ static bool promoteUniformPointerArgsToInReg(Module &M,
FuncChanged = Changed = true;
}
- if (FuncChanged) {
+ if (FuncChanged)
++NumPromotedInRegFuncs;
- PreservedAnalyses FuncPA;
- FuncPA.preserveSet<CFGAnalyses>();
- FAM.invalidate(F, FuncPA);
- }
}
return Changed;
@@ -136,7 +152,7 @@ PreservedAnalyses AMDGPUPromoteUniformArgsPass::run(Module &M,
ModuleAnalysisManager &AM) {
if (!EnablePromoteUniformArgs || !Triple(M.getTargetTriple()).isAMDGCN())
return PreservedAnalyses::all();
- if (!promoteUniformPointerArgsToInReg(M, AM))
+ if (!promoteUniformArgsToInReg(M))
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
index 39a884f14f479..ea6e55018e8bc 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-uniform-args.ll
@@ -1,340 +1,268 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -verify-each -mtriple=amdgpu9.0a-amd-amdhsa -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
+; RUN: opt -S -verify-each -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=amdgpu-promote-uniform-args < %s | FileCheck %s
-; A uniform pointer argument of an internal function, passed from a kernel,
-; is promoted to inreg (SGPR) on both the definition and the call site.
+ at gvar = addrspace(1) global float 0.000000e+00
+ at gptr = addrspace(1) global ptr null
+ at fnptr = global ptr null
+
+; Kernel scalar and pointer arguments are passed in SGPRs, so they are
+; trivially uniform and promoted. The workitem id is not trivially uniform.
-define internal fastcc void @callee_uniform(ptr %p, i32 %i) {
+define internal fastcc void @callee_uniform(ptr %p, i32 %i, float %f) {
; CHECK-LABEL: define internal fastcc void @callee_uniform(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-SAME: ptr inreg [[P:%.*]], i32 [[I:%.*]], float inreg [[F:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-NEXT: store float [[F]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
%g = getelementptr float, ptr %p, i32 %i
%v = load float, ptr %g
store float %v, ptr %p
+ store float %f, ptr %p
ret void
}
-define amdgpu_kernel void @k_uniform(ptr %p) {
+define amdgpu_kernel void @k_uniform(ptr %p, float %f) {
; CHECK-LABEL: define amdgpu_kernel void @k_uniform(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]], float [[F:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_uniform(ptr [[P]], i32 [[TID]])
+; CHECK-NEXT: call fastcc void @callee_uniform(ptr inreg [[P]], i32 [[TID]], float inreg [[F]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
- call fastcc void @callee_uniform(ptr %p, i32 %tid)
+ call fastcc void @callee_uniform(ptr %p, i32 %tid, float %f)
ret void
}
-; A divergent pointer operand (derived from the workitem id) must NOT be
-; promoted, because inreg would drop all but one lane's value.
+; Kernel integer and pointer arguments are both promoted.
-define internal fastcc void @callee_divergent(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_divergent(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_scalar(
+; CHECK-SAME: i32 inreg [[N:%.*]], ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- %v = load float, ptr %p
- store float %v, ptr %p
+ store i32 %n, ptr %p
ret void
}
-define amdgpu_kernel void @k_divergent(ptr %base) {
-; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
-; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
-; CHECK-NEXT: call fastcc void @callee_divergent(ptr [[PDIV]])
+define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_scalar(i32 inreg [[N]], ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- %pdiv = getelementptr float, ptr %base, i32 %tid
- call fastcc void @callee_divergent(ptr %pdiv)
+ call fastcc void @callee_scalar(i32 %n, ptr %p)
ret void
}
-; Private pointers name lane-private storage. Even if the pointer value itself
-; is uniform, the callee must not learn that the pointee is wave-uniform.
+; A constant pointer is trivially uniform.
-define internal fastcc void @callee_private(ptr addrspace(5) %p) {
-; CHECK-LABEL: define internal fastcc void @callee_private(
-; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
-; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+define internal fastcc void @callee_const(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_const(
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- store i32 1, ptr addrspace(5) %p
+ store float 0.000000e+00, ptr %p
ret void
}
-define amdgpu_kernel void @k_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_private() {
-; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: call fastcc void @callee_private(ptr addrspace(5) [[A]])
+define amdgpu_kernel void @k_const() {
+; CHECK-LABEL: define amdgpu_kernel void @k_const(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_const(ptr inreg addrspacecast (ptr addrspace(1) @gvar to ptr))
; CHECK-NEXT: ret void
;
- %a = alloca i32, addrspace(5)
- call fastcc void @callee_private(ptr addrspace(5) %a)
+ call fastcc void @callee_const(ptr addrspacecast (ptr addrspace(1) @gvar to ptr))
ret void
}
-; A flat pointer derived from private memory carries the same risk.
+; A constant integer is trivially uniform.
-define internal fastcc void @callee_flat_private(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_flat_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+define internal fastcc void @callee_const_i32(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_const_i32(
+; CHECK-SAME: i32 inreg [[N:%.*]], ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- store i32 1, ptr %p
+ store i32 %n, ptr %p
ret void
}
-define amdgpu_kernel void @k_flat_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_flat_private() {
-; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: [[F:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
-; CHECK-NEXT: call fastcc void @callee_flat_private(ptr [[F]])
+define amdgpu_kernel void @k_const_i32(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_const_i32(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_const_i32(i32 inreg 7, ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- %a = alloca i32, addrspace(5)
- %f = addrspacecast ptr addrspace(5) %a to ptr
- call fastcc void @callee_flat_private(ptr %f)
+ call fastcc void @callee_const_i32(i32 7, ptr %p)
ret void
}
-; A flat pointer that may be private on one path of a phi/select must not be
-; promoted, even though the other path is a benign global pointer.
+; C calling convention is eligible, not only fastcc.
-define internal fastcc void @callee_phi_private(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_phi_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store i32 1, ptr [[P]], align 4
+define internal void @callee_ccc(ptr %p) {
+; CHECK-LABEL: define internal void @callee_ccc(
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- store i32 1, ptr %p
+ store float 0.000000e+00, ptr %p
ret void
}
- at gvar = addrspace(1) global i32 0
-
-define amdgpu_kernel void @k_phi_private(i1 %c) {
-; CHECK-LABEL: define amdgpu_kernel void @k_phi_private(
-; CHECK-SAME: i1 [[C:%.*]]) {
-; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT: [[FA:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
-; CHECK-NEXT: [[FG:%.*]] = addrspacecast ptr addrspace(1) @gvar to ptr
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], ptr [[FA]], ptr [[FG]]
-; CHECK-NEXT: call fastcc void @callee_phi_private(ptr [[SEL]])
+define amdgpu_kernel void @k_ccc(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_ccc(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @callee_ccc(ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- %a = alloca i32, addrspace(5)
- %fa = addrspacecast ptr addrspace(5) %a to ptr
- %fg = addrspacecast ptr addrspace(1) @gvar to ptr
- %sel = select i1 %c, ptr %fa, ptr %fg
- call fastcc void @callee_phi_private(ptr %sel)
+ call void @callee_ccc(ptr %p)
ret void
}
-; A private pointer reached through a long getelementptr chain stays in the
-; private address space, so the address-space check still blocks it.
+; A divergent pointer (derived from the workitem id) must not be promoted.
-define internal fastcc void @callee_deep_private(ptr addrspace(5) %p) {
-; CHECK-LABEL: define internal fastcc void @callee_deep_private(
-; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
-; CHECK-NEXT: store i32 1, ptr addrspace(5) [[P]], align 4
+define internal fastcc void @callee_divergent(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_divergent(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- store i32 1, ptr addrspace(5) %p
+ %v = load float, ptr %p
+ store float %v, ptr %p
ret void
}
-define amdgpu_kernel void @k_deep_private() {
-; CHECK-LABEL: define amdgpu_kernel void @k_deep_private() {
-; CHECK-NEXT: [[A:%.*]] = alloca [64 x i32], align 4, addrspace(5)
-; CHECK-NEXT: [[G1:%.*]] = getelementptr i32, ptr addrspace(5) [[A]], i32 1
-; CHECK-NEXT: [[G2:%.*]] = getelementptr i32, ptr addrspace(5) [[G1]], i32 1
-; CHECK-NEXT: [[G3:%.*]] = getelementptr i32, ptr addrspace(5) [[G2]], i32 1
-; CHECK-NEXT: [[G4:%.*]] = getelementptr i32, ptr addrspace(5) [[G3]], i32 1
-; CHECK-NEXT: [[G5:%.*]] = getelementptr i32, ptr addrspace(5) [[G4]], i32 1
-; CHECK-NEXT: [[G6:%.*]] = getelementptr i32, ptr addrspace(5) [[G5]], i32 1
-; CHECK-NEXT: [[G7:%.*]] = getelementptr i32, ptr addrspace(5) [[G6]], i32 1
-; CHECK-NEXT: [[G8:%.*]] = getelementptr i32, ptr addrspace(5) [[G7]], i32 1
-; CHECK-NEXT: [[G9:%.*]] = getelementptr i32, ptr addrspace(5) [[G8]], i32 1
-; CHECK-NEXT: [[G10:%.*]] = getelementptr i32, ptr addrspace(5) [[G9]], i32 1
-; CHECK-NEXT: [[G11:%.*]] = getelementptr i32, ptr addrspace(5) [[G10]], i32 1
-; CHECK-NEXT: [[G12:%.*]] = getelementptr i32, ptr addrspace(5) [[G11]], i32 1
-; CHECK-NEXT: call fastcc void @callee_deep_private(ptr addrspace(5) [[G12]])
+define amdgpu_kernel void @k_divergent(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_divergent(
+; CHECK-SAME: ptr [[BASE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
+; CHECK-NEXT: call fastcc void @callee_divergent(ptr [[PDIV]])
; CHECK-NEXT: ret void
;
- %a = alloca [64 x i32], addrspace(5)
- %g1 = getelementptr i32, ptr addrspace(5) %a, i32 1
- %g2 = getelementptr i32, ptr addrspace(5) %g1, i32 1
- %g3 = getelementptr i32, ptr addrspace(5) %g2, i32 1
- %g4 = getelementptr i32, ptr addrspace(5) %g3, i32 1
- %g5 = getelementptr i32, ptr addrspace(5) %g4, i32 1
- %g6 = getelementptr i32, ptr addrspace(5) %g5, i32 1
- %g7 = getelementptr i32, ptr addrspace(5) %g6, i32 1
- %g8 = getelementptr i32, ptr addrspace(5) %g7, i32 1
- %g9 = getelementptr i32, ptr addrspace(5) %g8, i32 1
- %g10 = getelementptr i32, ptr addrspace(5) %g9, i32 1
- %g11 = getelementptr i32, ptr addrspace(5) %g10, i32 1
- %g12 = getelementptr i32, ptr addrspace(5) %g11, i32 1
- call fastcc void @callee_deep_private(ptr addrspace(5) %g12)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %pdiv = getelementptr float, ptr %base, i32 %tid
+ call fastcc void @callee_divergent(ptr %pdiv)
ret void
}
-; Callee-side guard: even a uniform flat pointer must not be promoted if the
-; callee reinterprets it as private (scratch), because that extracts a
-; lane-relative offset that is not wave-uniform.
+; Multiple uniform arguments are all promoted.
-define internal fastcc void @callee_casts_private(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_casts_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
-; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+define internal fastcc void @callee_multi(ptr %p0, ptr %p1, ptr %p2) {
+; CHECK-LABEL: define internal fastcc void @callee_multi(
+; CHECK-SAME: ptr inreg [[P0:%.*]], ptr inreg [[P1:%.*]], ptr inreg [[P2:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
- %q = addrspacecast ptr %p to ptr addrspace(5)
- store i32 1, ptr addrspace(5) %q
ret void
}
-define amdgpu_kernel void @k_casts_private(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_casts_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_casts_private(ptr [[P]])
+define amdgpu_kernel void @k_multi(ptr %p0, ptr %p1, ptr %p2) {
+; CHECK-LABEL: define amdgpu_kernel void @k_multi(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_multi(ptr inreg [[P0]], ptr inreg [[P1]], ptr inreg [[P2]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_casts_private(ptr %p)
+ call fastcc void @callee_multi(ptr %p0, ptr %p1, ptr %p2)
ret void
}
-; Positive control: a flat load/store through the argument is the common, safe
-; use and must still be promoted.
+; A loaded pointer is not trivially uniform.
-define internal fastcc void @callee_flat_load(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_flat_load(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+define internal fastcc void @callee_loaded(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_loaded(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- %v = load float, ptr %p
- store float %v, ptr %p
+ store float 0.000000e+00, ptr %p
ret void
}
-define amdgpu_kernel void @k_flat_load(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_flat_load(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_flat_load(ptr [[P]])
+define amdgpu_kernel void @k_loaded() {
+; CHECK-LABEL: define amdgpu_kernel void @k_loaded(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[P:%.*]] = load ptr, ptr addrspacecast (ptr addrspace(1) @gptr to ptr), align 8
+; CHECK-NEXT: call fastcc void @callee_loaded(ptr [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_flat_load(ptr %p)
+ %p = load ptr, ptr addrspacecast (ptr addrspace(1) @gptr to ptr)
+ call fastcc void @callee_loaded(ptr %p)
ret void
}
-; Multiple uniform pointer arguments are all promoted opportunistically.
-
-define internal fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
-; CHECK-LABEL: define internal fastcc void @callee_budget(
-; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
-; CHECK-NEXT: ret void
-;
- ptr %p4) {
- ret void
-}
+; A loaded integer is not trivially uniform.
-define amdgpu_kernel void @k_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3,
-; CHECK-LABEL: define amdgpu_kernel void @k_budget(
-; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_budget(ptr [[P0]], ptr [[P1]], ptr [[P2]], ptr [[P3]], ptr [[P4]])
+define internal fastcc void @callee_loaded_i32(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_loaded_i32(
+; CHECK-SAME: i32 [[N:%.*]], ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- ptr %p4) {
- call fastcc void @callee_budget(ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4)
+ store i32 %n, ptr %p
ret void
}
-; Trivial check: a loaded pointer is not AlwaysUniform, so the operand must not
-; be promoted even when the load address is wave-uniform.
-
-define internal fastcc void @callee_tti_flatload(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_tti_flatload(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+define amdgpu_kernel void @k_loaded_i32(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_loaded_i32(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[N:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: call fastcc void @callee_loaded_i32(i32 [[N]], ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- store float 0.000000e+00, ptr %p
+ %n = load i32, ptr %p
+ call fastcc void @callee_loaded_i32(i32 %n, ptr %p)
ret void
}
- at gptr = addrspace(1) global ptr null
+; External / linkonce_odr linkage: the ABI is observable outside this module.
-define amdgpu_kernel void @k_tti_flatload() {
-; CHECK-LABEL: define amdgpu_kernel void @k_tti_flatload() {
-; CHECK-NEXT: [[FLATG:%.*]] = addrspacecast ptr addrspace(1) @gptr to ptr
-; CHECK-NEXT: [[P:%.*]] = load ptr, ptr [[FLATG]], align 8
-; CHECK-NEXT: call fastcc void @callee_tti_flatload(ptr [[P]])
+define fastcc void @callee_external(ptr %p) {
+; CHECK-LABEL: define fastcc void @callee_external(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
- %flatg = addrspacecast ptr addrspace(1) @gptr to ptr
- %p = load ptr, ptr %flatg
- call fastcc void @callee_tti_flatload(ptr %p)
ret void
}
-; External linkage: all call sites are not necessarily visible, so the ABI
-; must not be changed.
-
-define fastcc void @callee_external(ptr %p) {
-; CHECK-LABEL: define fastcc void @callee_external(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+define linkonce_odr fastcc void @callee_linkonce_odr(ptr %p) {
+; CHECK-LABEL: define linkonce_odr fastcc void @callee_linkonce_odr(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
- %v = load float, ptr %p
- store float %v, ptr %p
ret void
}
define amdgpu_kernel void @k_external(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_external(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_external(ptr [[P]])
+; CHECK-NEXT: call fastcc void @callee_linkonce_odr(ptr [[P]])
; CHECK-NEXT: ret void
;
call fastcc void @callee_external(ptr %p)
+ call fastcc void @callee_linkonce_odr(ptr %p)
ret void
}
-; Address-taken internal function: an indirect call we cannot see could pass a
-; divergent value, so do not promote.
+; Address taken via a store.
define internal fastcc void @callee_addrtaken(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_addrtaken(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
- %v = load float, ptr %p
- store float %v, ptr %p
ret void
}
- at fnptr = global ptr null
-
define amdgpu_kernel void @k_addrtaken(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_addrtaken(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store ptr @callee_addrtaken, ptr @fnptr, align 8
; CHECK-NEXT: call fastcc void @callee_addrtaken(ptr [[P]])
; CHECK-NEXT: ret void
@@ -344,13 +272,37 @@ define amdgpu_kernel void @k_addrtaken(ptr %p) {
ret void
}
-; Arguments with ABI-affecting attributes must not be promoted.
+; Address escaping as a call argument.
+
+define internal fastcc void @callee_addr_as_operand(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_addr_as_operand(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: ret void
+;
+ ret void
+}
+
+define amdgpu_kernel void @k_addr_as_operand(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_addr_as_operand(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_addr_as_operand(ptr [[P]])
+; CHECK-NEXT: call void @sink(ptr @callee_addr_as_operand)
+; CHECK-NEXT: ret void
+;
+ call fastcc void @callee_addr_as_operand(ptr %p)
+ call void @sink(ptr @callee_addr_as_operand)
+ ret void
+}
+
+declare void @sink(ptr)
+
+; byref / hidden arguments keep their ABI.
%struct.Foo = type { i32 }
define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
; CHECK-LABEL: define internal fastcc void @callee_byref(
-; CHECK-SAME: ptr byref([[STRUCT_FOO:%.*]]) [[P:%.*]]) {
+; CHECK-SAME: ptr byref([[STRUCT_FOO:%.*]]) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
ret void
@@ -358,7 +310,7 @@ define internal fastcc void @callee_byref(ptr byref(%struct.Foo) %p) {
define amdgpu_kernel void @k_byref(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_byref(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_byref(ptr byref([[STRUCT_FOO:%.*]]) [[P]])
; CHECK-NEXT: ret void
;
@@ -366,47 +318,29 @@ define amdgpu_kernel void @k_byref(ptr %p) {
ret void
}
-define internal fastcc ptr @callee_returned(ptr returned %p) {
-; CHECK-LABEL: define internal fastcc ptr @callee_returned(
-; CHECK-SAME: ptr returned [[P:%.*]]) {
-; CHECK-NEXT: ret ptr [[P]]
-;
- ret ptr %p
-}
-
-define amdgpu_kernel void @k_returned(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_returned(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call fastcc ptr @callee_returned(ptr returned [[P]])
-; CHECK-NEXT: store ptr [[R]], ptr [[P]], align 8
-; CHECK-NEXT: ret void
-;
- %r = call fastcc ptr @callee_returned(ptr returned %p)
- store ptr %r, ptr %p
- ret void
-}
+; inreg is mutually exclusive with nest.
-define internal fastcc void @callee_swiftasync(ptr swiftasync %p) {
-; CHECK-LABEL: define internal fastcc void @callee_swiftasync(
-; CHECK-SAME: ptr swiftasync [[P:%.*]]) {
+define internal fastcc void @callee_nest(ptr nest %p) {
+; CHECK-LABEL: define internal fastcc void @callee_nest(
+; CHECK-SAME: ptr nest [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
ret void
}
-define amdgpu_kernel void @k_swiftasync(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_swiftasync(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_swiftasync(ptr swiftasync [[P]])
+define amdgpu_kernel void @k_nest(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_nest(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_nest(ptr nest [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_swiftasync(ptr swiftasync %p)
+ call fastcc void @callee_nest(ptr nest %p)
ret void
}
define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
; CHECK-LABEL: define internal fastcc void @callee_hidden(
-; CHECK-SAME: ptr "amdgpu-hidden-argument" [[P:%.*]]) {
+; CHECK-SAME: ptr "amdgpu-hidden-argument" [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
ret void
@@ -414,7 +348,7 @@ define internal fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" %p) {
define amdgpu_kernel void @k_hidden(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_hidden(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_hidden(ptr "amdgpu-hidden-argument" [[P]])
; CHECK-NEXT: ret void
;
@@ -422,47 +356,22 @@ define amdgpu_kernel void @k_hidden(ptr %p) {
ret void
}
-; Non-pointer uniform arguments are out of scope for this pass.
-
-define internal fastcc void @callee_scalar(i32 %n, ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_scalar(
-; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store i32 %n, ptr %p
- ret void
-}
-
-define amdgpu_kernel void @k_scalar(i32 %n, ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_scalar(
-; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_scalar(i32 [[N]], ptr [[P]])
-; CHECK-NEXT: ret void
-;
-; The pointer is still promoted, the scalar is not.
- call fastcc void @callee_scalar(i32 %n, ptr %p)
- ret void
-}
-
-; Mixed call sites: one uniform, one divergent. A single divergent operand must
-; block promotion, since the definition is shared by all callers.
+; Mixed call sites: one uniform, one divergent. The shared definition is not
+; promoted.
define internal fastcc void @callee_mixed(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_mixed(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[P]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- %v = load float, ptr %p
- store float %v, ptr %p
+ store float 0.000000e+00, ptr %p
ret void
}
define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_uniform(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[P]])
; CHECK-NEXT: ret void
;
@@ -472,7 +381,7 @@ define amdgpu_kernel void @k_mixed_uniform(ptr %p) {
define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_mixed_divergent(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-SAME: ptr [[BASE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
; CHECK-NEXT: call fastcc void @callee_mixed(ptr [[PDIV]])
@@ -484,95 +393,46 @@ define amdgpu_kernel void @k_mixed_divergent(ptr %base) {
ret void
}
-; Multi-hop chain: uniformity propagates from the kernel through each internal
-; function to a fixpoint, so every hop's closure pointer is promoted.
-
-define internal fastcc void @chain_leaf(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @chain_leaf(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define internal fastcc void @chain_mid(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @chain_mid(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_leaf(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @chain_leaf(ptr %p)
- ret void
-}
-
-define internal fastcc void @chain_top(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @chain_top(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_mid(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @chain_mid(ptr %p)
- ret void
-}
+; Mixed call sites for a scalar: one uniform, one divergent.
-define amdgpu_kernel void @k_chain(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_chain(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @chain_top(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @chain_top(ptr %p)
- ret void
-}
-
-; Fixpoint must converge regardless of the order the functions appear in the
-; module (callees defined before/after their callers) and across a diamond.
-
-define internal fastcc void @scram_a(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @scram_a(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_b(ptr [[P]])
+define internal fastcc void @callee_mixed_i32(i32 %n, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_mixed_i32(
+; CHECK-SAME: i32 [[N:%.*]], ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store i32 [[N]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
- call fastcc void @scram_b(ptr %p)
+ store i32 %n, ptr %p
ret void
}
-define internal fastcc void @scram_c(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @scram_c(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+define amdgpu_kernel void @k_mixed_i32_uniform(i32 %n, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_i32_uniform(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_mixed_i32(i32 [[N]], ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- store float 0.000000e+00, ptr %p
+ call fastcc void @callee_mixed_i32(i32 %n, ptr %p)
ret void
}
-define internal fastcc void @scram_b(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @scram_b(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_c(ptr [[P]])
+define amdgpu_kernel void @k_mixed_i32_divergent(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_mixed_i32_divergent(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: call fastcc void @callee_mixed_i32(i32 [[TID]], ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @scram_c(ptr %p)
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ call fastcc void @callee_mixed_i32(i32 %tid, ptr %p)
ret void
}
-define amdgpu_kernel void @k_scram(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_scram(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @scram_a(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @scram_a(ptr %p)
- ret void
-}
+; The operand at @chain_leaf is an internal-function argument without inreg, so
+; it is not trivially uniform. @chain_top's operand is a kernel argument.
-define internal fastcc void @diam_bot(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @diam_bot(
-; CHECK-SAME: ptr [[P:%.*]]) {
+define internal fastcc void @chain_leaf(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @chain_leaf(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -580,54 +440,31 @@ define internal fastcc void @diam_bot(ptr %p) {
ret void
}
-define internal fastcc void @diam_l(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @diam_l(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_bot(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @diam_bot(ptr %p)
- ret void
-}
-
-define internal fastcc void @diam_r(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @diam_r(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_bot(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @diam_bot(ptr %p)
- ret void
-}
-
-define internal fastcc void @diam_top(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @diam_top(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_l(ptr [[P]])
-; CHECK-NEXT: call fastcc void @diam_r(ptr [[P]])
+define internal fastcc void @chain_top(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @chain_top(
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @chain_leaf(ptr [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @diam_l(ptr %p)
- call fastcc void @diam_r(ptr %p)
+ call fastcc void @chain_leaf(ptr %p)
ret void
}
-define amdgpu_kernel void @k_diam(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_diam(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @diam_top(ptr [[P]])
+define amdgpu_kernel void @k_chain(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_chain(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @chain_top(ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @diam_top(ptr %p)
+ call fastcc void @chain_top(ptr %p)
ret void
}
-; An argument that is already inreg must be left untouched (no double attribute,
-; no crash).
+; Already-inreg arguments are left alone.
define internal fastcc void @callee_already(ptr inreg %p) {
; CHECK-LABEL: define internal fastcc void @callee_already(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -637,7 +474,7 @@ define internal fastcc void @callee_already(ptr inreg %p) {
define amdgpu_kernel void @k_already(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_already(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_already(ptr inreg [[P]])
; CHECK-NEXT: ret void
;
@@ -645,12 +482,11 @@ define amdgpu_kernel void @k_already(ptr %p) {
ret void
}
-; Invoke call sites are not audited for inreg ABI consistency under exceptional
-; control flow, so promotion is skipped.
+; Invoke call sites block promotion.
define internal fastcc void @callee_invoke(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_invoke(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -660,7 +496,7 @@ define internal fastcc void @callee_invoke(ptr %p) {
define amdgpu_kernel void @k_invoke(ptr %p) personality ptr null {
; CHECK-LABEL: define amdgpu_kernel void @k_invoke(
-; CHECK-SAME: ptr [[P:%.*]]) personality ptr null {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] personality ptr null {
; CHECK-NEXT: invoke fastcc void @callee_invoke(ptr [[P]])
; CHECK-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD:.*]]
; CHECK: [[CONT]]:
@@ -681,63 +517,11 @@ lpad:
ret void
}
-; Indirect call through a bitcast of the function pointer: the callee is not a
-; direct reference to @callee_bitcast, so the pass cannot prove all call sites.
-
-define internal fastcc void @callee_bitcast(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_bitcast(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define amdgpu_kernel void @k_bitcast(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_bitcast(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[FN:%.*]] = bitcast ptr @callee_bitcast to ptr
-; CHECK-NEXT: call void [[FN]](ptr [[P]])
-; CHECK-NEXT: ret void
-;
- %fn = bitcast ptr @callee_bitcast to ptr
- call void %fn(ptr %p)
- ret void
-}
-
-; A function stored into a global (non-call use) is not eligible even if there
-; is also a direct call the pass can see.
-
-define internal fastcc void @callee_stored(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_stored(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
- at fn_slot = global ptr null
-
-define amdgpu_kernel void @k_stored(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_stored(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store ptr @callee_stored, ptr @fn_slot, align 8
-; CHECK-NEXT: call fastcc void @callee_stored(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- store ptr @callee_stored, ptr @fn_slot
- call fastcc void @callee_stored(ptr %p)
- ret void
-}
-
-; musttail forwarding of an argument blocks inreg promotion for that argument.
+; musttail as a call *to* F blocks changing F's ABI.
define internal fastcc void @tail_target(ptr %q) {
; CHECK-LABEL: define internal fastcc void @tail_target(
-; CHECK-SAME: ptr [[Q:%.*]]) {
+; CHECK-SAME: ptr [[Q:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
ret void
@@ -745,7 +529,7 @@ define internal fastcc void @tail_target(ptr %q) {
define internal fastcc void @callee_musttail_forward(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_musttail_forward(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: musttail call fastcc void @tail_target(ptr [[P]])
; CHECK-NEXT: ret void
;
@@ -755,7 +539,7 @@ define internal fastcc void @callee_musttail_forward(ptr %p) {
define amdgpu_kernel void @k_musttail_forward(ptr %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_musttail_forward(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_musttail_forward(ptr [[P]])
; CHECK-NEXT: ret void
;
@@ -763,67 +547,36 @@ define amdgpu_kernel void @k_musttail_forward(ptr %p) {
ret void
}
-; inlinehint and alwaysinline callees behave the same for inreg promotion when
-; the callee remains an out-of-line call (the motivating Kokkos case).
-
-define internal fastcc void @callee_inlinehint_uniform(ptr %p, i32 %i) #0 {
-; CHECK-LABEL: define internal fastcc void @callee_inlinehint_uniform(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- %g = getelementptr float, ptr %p, i32 %i
- %v = load float, ptr %g
- store float %v, ptr %p
- ret void
-}
+; musttail in the body blocks promoting any argument of that function, including
+; one that is not forwarded to the tail call.
-define amdgpu_kernel void @k_inlinehint_uniform(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint_uniform(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_inlinehint_uniform(ptr [[P]], i32 [[TID]])
-; CHECK-NEXT: ret void
-;
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- call fastcc void @callee_inlinehint_uniform(ptr %p, i32 %tid)
- ret void
-}
+declare fastcc void @mt_abi_target(ptr, ptr)
-define internal fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %i) #1 {
-; CHECK-LABEL: define internal fastcc void @callee_alwaysinline_uniform(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[G:%.*]] = getelementptr float, ptr [[P]], i32 [[I]]
-; CHECK-NEXT: [[V:%.*]] = load float, ptr [[G]], align 4
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4
+define internal fastcc void @callee_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define internal fastcc void @callee_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: musttail call fastcc void @mt_abi_target(ptr [[Q]], ptr [[Q]])
; CHECK-NEXT: ret void
;
- %g = getelementptr float, ptr %p, i32 %i
- %v = load float, ptr %g
- store float %v, ptr %p
+ musttail call fastcc void @mt_abi_target(ptr %q, ptr %q)
ret void
}
-define amdgpu_kernel void @k_alwaysinline_uniform(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_alwaysinline_uniform(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: call fastcc void @callee_alwaysinline_uniform(ptr [[P]], i32 [[TID]])
+define amdgpu_kernel void @k_musttail_abi(ptr %p, ptr %q) {
+; CHECK-LABEL: define amdgpu_kernel void @k_musttail_abi(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_musttail_abi(ptr [[P]], ptr [[Q]])
; CHECK-NEXT: ret void
;
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- call fastcc void @callee_alwaysinline_uniform(ptr %p, i32 %tid)
+ call fastcc void @callee_musttail_abi(ptr %p, ptr %q)
ret void
}
-; readfirstlane produces an always-uniform value (per TargetTransformInfo), so a
-; pointer laundered through it is promoted even though its input is divergent.
+; readfirstlane in the same block as the call is trivially uniform.
define internal fastcc void @callee_readfirstlane(ptr %p) {
; CHECK-LABEL: define internal fastcc void @callee_readfirstlane(
-; CHECK-SAME: ptr inreg [[P:%.*]]) {
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -833,7 +586,7 @@ define internal fastcc void @callee_readfirstlane(ptr %p) {
define amdgpu_kernel void @k_readfirstlane(ptr %base) {
; CHECK-LABEL: define amdgpu_kernel void @k_readfirstlane(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-SAME: ptr [[BASE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
@@ -847,12 +600,12 @@ define amdgpu_kernel void @k_readfirstlane(ptr %base) {
ret void
}
-; A single callee reached from two kernels through different uniform sources (a
-; kernel SGPR argument and a readfirstlane result) is still promoted.
+; readfirstlane in a different block is not trivially uniform (temporal
+; divergence).
-define internal fastcc void @multi_src(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @multi_src(
-; CHECK-SAME: ptr [[P:%.*]]) {
+define internal fastcc void @callee_rfl_otherbb(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_rfl_otherbb(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
@@ -860,267 +613,31 @@ define internal fastcc void @multi_src(ptr %p) {
ret void
}
-define amdgpu_kernel void @k_multi_sgpr(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_multi_sgpr(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @multi_src(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @multi_src(ptr %p)
- ret void
-}
-
-define amdgpu_kernel void @k_multi_readfirstlane(ptr %base) {
-; CHECK-LABEL: define amdgpu_kernel void @k_multi_readfirstlane(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
+define amdgpu_kernel void @k_rfl_otherbb(ptr %base) {
+; CHECK-LABEL: define amdgpu_kernel void @k_rfl_otherbb(
+; CHECK-SAME: ptr [[BASE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
-; CHECK-NEXT: call fastcc void @multi_src(ptr [[PUNI]])
+; CHECK-NEXT: br label %[[BB:.*]]
+; CHECK: [[BB]]:
+; CHECK-NEXT: call fastcc void @callee_rfl_otherbb(ptr [[PUNI]])
; CHECK-NEXT: ret void
;
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%pdiv = getelementptr float, ptr %base, i32 %tid
%puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
- call fastcc void @multi_src(ptr %puni)
- ret void
-}
-
-; Negative counterpart of the multi-source shape: one call site passes a uniform
-; (readfirstlane) operand and the other passes a divergent one. A single
-; divergent source blocks promotion of the shared definition, so no inreg appears
-; on the definition or at either call site.
-
-define internal fastcc void @multi_src_mixed(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @multi_src_mixed(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define amdgpu_kernel void @k_multi_mixed_uniform(ptr %base) {
-; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_uniform(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
-; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
-; CHECK-NEXT: [[PUNI:%.*]] = call ptr @llvm.amdgcn.readfirstlane.p0(ptr [[PDIV]])
-; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PUNI]])
-; CHECK-NEXT: ret void
-;
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- %pdiv = getelementptr float, ptr %base, i32 %tid
- %puni = call ptr @llvm.amdgcn.readfirstlane.p0(ptr %pdiv)
- call fastcc void @multi_src_mixed(ptr %puni)
- ret void
-}
-
-define amdgpu_kernel void @k_multi_mixed_divergent(ptr %base) {
-; CHECK-LABEL: define amdgpu_kernel void @k_multi_mixed_divergent(
-; CHECK-SAME: ptr [[BASE:%.*]]) {
-; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; CHECK-NEXT: [[PDIV:%.*]] = getelementptr float, ptr [[BASE]], i32 [[TID]]
-; CHECK-NEXT: call fastcc void @multi_src_mixed(ptr [[PDIV]])
-; CHECK-NEXT: ret void
-;
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- %pdiv = getelementptr float, ptr %base, i32 %tid
- call fastcc void @multi_src_mixed(ptr %pdiv)
- ret void
-}
-
-; Self-recursive callee that forwards its own pointer argument: the fixpoint must
-; terminate. Promotion is conservatively skipped because the recursive call site
-; forwards the argument back to itself (the cycle cannot be proven uniform here).
-
-define internal fastcc void @self_rec(ptr %p, i32 %n) {
-; CHECK-LABEL: define internal fastcc void @self_rec(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
-; CHECK: [[REC]]:
-; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 [[N1]])
-; CHECK-NEXT: br label %[[END]]
-; CHECK: [[END]]:
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- %c = icmp ne i32 %n, 0
- br i1 %c, label %rec, label %end
-rec:
- %n1 = sub i32 %n, 1
- call fastcc void @self_rec(ptr %p, i32 %n1)
- br label %end
-end:
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define amdgpu_kernel void @k_self_rec(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_self_rec(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @self_rec(ptr [[P]], i32 10)
-; CHECK-NEXT: ret void
-;
- call fastcc void @self_rec(ptr %p, i32 10)
- ret void
-}
-
-; Mutually recursive callees that forward the pointer around a cycle: likewise
-; must terminate and are conservatively left unpromoted.
-
-define internal fastcc void @mutual_a(ptr %p, i32 %n) {
-; CHECK-LABEL: define internal fastcc void @mutual_a(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
-; CHECK: [[REC]]:
-; CHECK-NEXT: [[N1:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: call fastcc void @mutual_b(ptr [[P]], i32 [[N1]])
-; CHECK-NEXT: br label %[[END]]
-; CHECK: [[END]]:
-; CHECK-NEXT: ret void
-;
- %c = icmp ne i32 %n, 0
- br i1 %c, label %rec, label %end
-rec:
- %n1 = sub i32 %n, 1
- call fastcc void @mutual_b(ptr %p, i32 %n1)
- br label %end
-end:
- ret void
-}
-
-define internal fastcc void @mutual_b(ptr %p, i32 %n) {
-; CHECK-LABEL: define internal fastcc void @mutual_b(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[N]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[REC:.*]], label %[[END:.*]]
-; CHECK: [[REC]]:
-; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 [[N]])
-; CHECK-NEXT: br label %[[END]]
-; CHECK: [[END]]:
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- %c = icmp ne i32 %n, 0
- br i1 %c, label %rec, label %end
-rec:
- call fastcc void @mutual_a(ptr %p, i32 %n)
- br label %end
-end:
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define amdgpu_kernel void @k_mutual(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_mutual(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @mutual_a(ptr [[P]], i32 5)
-; CHECK-NEXT: ret void
-;
- call fastcc void @mutual_a(ptr %p, i32 5)
- ret void
-}
-
-; An intermediate caller that is address-taken could be reached indirectly with a
-; divergent pointer, so a leaf reached only through it must NOT be promoted even
-; though the visible direct call is uniform.
-
- at addr_slot = global ptr null
-
-define internal fastcc void @ci_leaf(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @ci_leaf(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define internal fastcc void @ci_mid(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @ci_mid(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @ci_leaf(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @ci_leaf(ptr %p)
- ret void
-}
-
-define amdgpu_kernel void @k_ci(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_ci(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store ptr @ci_mid, ptr @addr_slot, align 8
-; CHECK-NEXT: call fastcc void @ci_mid(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- store ptr @ci_mid, ptr @addr_slot
- call fastcc void @ci_mid(ptr %p)
+ br label %bb
+bb:
+ call fastcc void @callee_rfl_otherbb(ptr %puni)
ret void
}
-; The same shared leaf reached through two intermediates: one address-taken, one
-; clean. The address-taken path taints the leaf (a single unprovable call site
-; blocks the shared definition), so @ci2_leaf is NOT promoted. The clean
-; intermediate @ci2_mid_clean is still promoted on its own, because its only call
-; site is the uniform kernel call and promotion is decided per callee.
-
- at addr_slot2 = global ptr null
-
-define internal fastcc void @ci2_leaf(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @ci2_leaf(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
-; CHECK-NEXT: ret void
-;
- store float 0.000000e+00, ptr %p
- ret void
-}
-
-define internal fastcc void @ci2_mid_at(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @ci2_mid_at(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @ci2_leaf(ptr %p)
- ret void
-}
-
-define internal fastcc void @ci2_mid_clean(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @ci2_mid_clean(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @ci2_leaf(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- call fastcc void @ci2_leaf(ptr %p)
- ret void
-}
-
-define amdgpu_kernel void @k_ci2(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_ci2(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: store ptr @ci2_mid_at, ptr @addr_slot2, align 8
-; CHECK-NEXT: call fastcc void @ci2_mid_at(ptr [[P]])
-; CHECK-NEXT: call fastcc void @ci2_mid_clean(ptr [[P]])
-; CHECK-NEXT: ret void
-;
- store ptr @ci2_mid_at, ptr @addr_slot2
- call fastcc void @ci2_mid_at(ptr %p)
- call fastcc void @ci2_mid_clean(ptr %p)
- ret void
-}
-
-; A vector-of-pointers argument is not a scalar pointer and is out of scope.
+; Vectors are not promoted.
define internal fastcc void @callee_vecptr(<2 x ptr> %p) {
; CHECK-LABEL: define internal fastcc void @callee_vecptr(
-; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
ret void
@@ -1128,7 +645,7 @@ define internal fastcc void @callee_vecptr(<2 x ptr> %p) {
define amdgpu_kernel void @k_vecptr(<2 x ptr> %p) {
; CHECK-LABEL: define amdgpu_kernel void @k_vecptr(
-; CHECK-SAME: <2 x ptr> [[P:%.*]]) {
+; CHECK-SAME: <2 x ptr> [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: call fastcc void @callee_vecptr(<2 x ptr> [[P]])
; CHECK-NEXT: ret void
;
@@ -1136,130 +653,109 @@ define amdgpu_kernel void @k_vecptr(<2 x ptr> %p) {
ret void
}
-; A musttail call requires the enclosing function's parameter ABI attributes to
-; match the call positionally, regardless of which values are forwarded. Promoting
-; even an unrelated uniform argument (@p, which is not passed to the musttail
-; call) would break that contract, so no argument of a function containing a
-; musttail call is promoted.
-
-declare fastcc void @mt_abi_target(ptr, ptr)
-
-define internal fastcc void @callee_musttail_abi(ptr %p, ptr %q) {
-; CHECK-LABEL: define internal fastcc void @callee_musttail_abi(
-; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
-; CHECK-NEXT: musttail call fastcc void @mt_abi_target(ptr [[Q]], ptr [[Q]])
+define internal fastcc void @callee_vecint(<2 x i32> %v, ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_vecint(
+; CHECK-SAME: <2 x i32> [[V:%.*]], ptr inreg [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x i32> [[V]], ptr [[P]], align 8
; CHECK-NEXT: ret void
;
- musttail call fastcc void @mt_abi_target(ptr %q, ptr %q)
+ store <2 x i32> %v, ptr %p
ret void
}
-define amdgpu_kernel void @k_musttail_abi(ptr %p, ptr %q) {
-; CHECK-LABEL: define amdgpu_kernel void @k_musttail_abi(
-; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_musttail_abi(ptr [[P]], ptr [[Q]])
+define amdgpu_kernel void @k_vecint(<2 x i32> %v, ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_vecint(
+; CHECK-SAME: <2 x i32> [[V:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_vecint(<2 x i32> [[V]], ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_musttail_abi(ptr %p, ptr %q)
+ call fastcc void @callee_vecint(<2 x i32> %v, ptr %p)
ret void
}
-; A freeze between the argument and a private addrspacecast must not hide the
-; private reinterpretation from the callee-side guard.
+; optnone on the caller does not block updating the call site.
-define internal fastcc void @callee_freeze_private(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_freeze_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[F:%.*]] = freeze ptr [[P]]
-; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[F]] to ptr addrspace(5)
-; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+define internal fastcc void @callee_optnone_caller(ptr %p) {
+; CHECK-LABEL: define internal fastcc void @callee_optnone_caller(
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: ret void
;
- %f = freeze ptr %p
- %q = addrspacecast ptr %f to ptr addrspace(5)
- store i32 1, ptr addrspace(5) %q
ret void
}
-define amdgpu_kernel void @k_freeze_private(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_freeze_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_freeze_private(ptr [[P]])
+define amdgpu_kernel void @k_optnone_caller(ptr %p) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @k_optnone_caller(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: call fastcc void @callee_optnone_caller(ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_freeze_private(ptr %p)
+ call fastcc void @callee_optnone_caller(ptr %p)
ret void
}
-; ptrmask likewise forwards the pointer; a private cast behind it must still be
-; detected by the callee-side guard.
+; optnone / naked / noipa on the callee forbid a signature change.
-define internal fastcc void @callee_ptrmask_private(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_ptrmask_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[M:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -16)
-; CHECK-NEXT: [[Q:%.*]] = addrspacecast ptr [[M]] to ptr addrspace(5)
-; CHECK-NEXT: store i32 1, ptr addrspace(5) [[Q]], align 4
+define internal fastcc void @callee_optnone_callee(ptr %p) #0 {
+; CHECK-LABEL: define internal fastcc void @callee_optnone_callee(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: ret void
;
- %m = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -16)
- %q = addrspacecast ptr %m to ptr addrspace(5)
- store i32 1, ptr addrspace(5) %q
ret void
}
-define amdgpu_kernel void @k_ptrmask_private(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_ptrmask_private(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_ptrmask_private(ptr [[P]])
+define internal fastcc void @callee_naked(ptr %p) #1 {
+; CHECK-LABEL: define internal fastcc void @callee_naked(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: ret void
;
- call fastcc void @callee_ptrmask_private(ptr %p)
ret void
}
-; optnone on the caller: ABI attributes must not be changed.
-define internal fastcc void @callee_optnone_caller(ptr %p) {
-; CHECK-LABEL: define internal fastcc void @callee_optnone_caller(
-; CHECK-SAME: ptr [[P:%.*]]) {
+define internal fastcc void @callee_noipa(ptr %p) #2 {
+; CHECK-LABEL: define internal fastcc void @callee_noipa(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR3:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}
-define amdgpu_kernel void @k_optnone_caller(ptr %p) #2 {
-; CHECK-LABEL: define amdgpu_kernel void @k_optnone_caller(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: call fastcc void @callee_optnone_caller(ptr [[P]])
+define amdgpu_kernel void @k_no_signature_change(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_no_signature_change(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_optnone_callee(ptr [[P]])
+; CHECK-NEXT: call fastcc void @callee_naked(ptr [[P]])
+; CHECK-NEXT: call fastcc void @callee_noipa(ptr [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_optnone_caller(ptr %p)
+ call fastcc void @callee_optnone_callee(ptr %p)
+ call fastcc void @callee_naked(ptr %p)
+ call fastcc void @callee_noipa(ptr %p)
ret void
}
-; optnone on the callee itself.
-define internal fastcc void @callee_optnone_callee(ptr %p) #2 {
-; CHECK-LABEL: define internal fastcc void @callee_optnone_callee(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR2]] {
+; inlinehint does not affect promotion.
+
+define internal fastcc void @callee_inlinehint(ptr %p) #3 {
+; CHECK-LABEL: define internal fastcc void @callee_inlinehint(
+; CHECK-SAME: ptr inreg [[P:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; CHECK-NEXT: ret void
;
+ store float 0.000000e+00, ptr %p
ret void
}
-define amdgpu_kernel void @k_optnone_callee(ptr %p) {
-; CHECK-LABEL: define amdgpu_kernel void @k_optnone_callee(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: call fastcc void @callee_optnone_callee(ptr [[P]])
+define amdgpu_kernel void @k_inlinehint(ptr %p) {
+; CHECK-LABEL: define amdgpu_kernel void @k_inlinehint(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call fastcc void @callee_inlinehint(ptr inreg [[P]])
; CHECK-NEXT: ret void
;
- call fastcc void @callee_optnone_callee(ptr %p)
+ call fastcc void @callee_inlinehint(ptr %p)
ret void
}
-attributes #0 = { inlinehint }
-attributes #1 = { alwaysinline }
-attributes #2 = { noinline optnone }
-
-declare i32 @llvm.amdgcn.workitem.id.x()
-declare ptr @llvm.amdgcn.readfirstlane.p0(ptr)
-declare ptr @llvm.ptrmask.p0.i64(ptr, i64)
+attributes #0 = { noinline optnone }
+attributes #1 = { naked }
+attributes #2 = { noipa }
+attributes #3 = { inlinehint }
More information about the llvm-commits
mailing list