<div dir="ltr">This change broke the sanitizer bots with the following failure:<div><div>/b/sanitizer-x86_64-linux-fast/build/llvm/tools/clang/test/CodeGen/backend-unsupported-error.ll:6:10: error: expected string not found in input</div><div>; CHECK: error: test.c:2:20: in function bar i32 (): unsupported call to function foo.2</div><div> ^</div><div><stdin>:1:1: note: scanning from here</div><div>'-fp32-denormals' is not a recognized feature for this target (ignoring feature)</div><div>^</div><div><stdin>:4:1: note: possible intended match here</div><div>error: test.c:3:20: in function bar i32 (): unsupported call to function bar</div><div>^</div><div><br></div><div>I have reverted it in r336623.</div><br><div class="gmail_quote"><div dir="ltr">On Mon, Jul 9, 2018 at 12:27 PM Matt Arsenault via llvm-commits <<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: arsenm<br>
Date: Mon Jul 9 12:22:22 2018<br>
New Revision: 336587<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=336587&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=336587&view=rev</a><br>
Log:<br>
AMDGPU: Force inlining if LDS global address is used<br>
<br>
These won't work for the forseeable future. These aren't allowed<br>
from OpenCL, but IPO optimizations can make them appear.<br>
<br>
Also directly set the attributes on functions, regardless<br>
of the linkage rather than cloning functions like before.<br>
<br>
Added:<br>
llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll<br>
Modified:<br>
llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp<br>
llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp<br>
llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h<br>
llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll<br>
llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll<br>
<br>
Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp?rev=336587&r1=336586&r2=336587&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp?rev=336587&r1=336586&r2=336587&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp (original)<br>
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp Mon Jul 9 12:22:22 2018<br>
@@ -14,6 +14,9 @@<br>
//===----------------------------------------------------------------------===//<br>
<br>
#include "AMDGPU.h"<br>
+#include "AMDGPUTargetMachine.h"<br>
+#include "Utils/AMDGPUBaseInfo.h"<br>
+#include "llvm/ADT/SmallPtrSet.h"<br>
#include "llvm/IR/Module.h"<br>
#include "llvm/Transforms/Utils/Cloning.h"<br>
<br>
@@ -30,13 +33,18 @@ static cl::opt<bool> StressCalls(<br>
class AMDGPUAlwaysInline : public ModulePass {<br>
bool GlobalOpt;<br>
<br>
+ void recursivelyVisitUsers(GlobalValue &GV,<br>
+ SmallPtrSetImpl<Function *> &FuncsToAlwaysInline);<br>
public:<br>
static char ID;<br>
<br>
AMDGPUAlwaysInline(bool GlobalOpt = false) :<br>
ModulePass(ID), GlobalOpt(GlobalOpt) { }<br>
bool runOnModule(Module &M) override;<br>
- StringRef getPassName() const override { return "AMDGPU Always Inline Pass"; }<br>
+<br>
+ void getAnalysisUsage(AnalysisUsage &AU) const override {<br>
+ AU.setPreservesAll();<br>
+ }<br>
};<br>
<br>
} // End anonymous namespace<br>
@@ -46,15 +54,53 @@ INITIALIZE_PASS(AMDGPUAlwaysInline, "amd<br>
<br>
char AMDGPUAlwaysInline::ID = 0;<br>
<br>
+void AMDGPUAlwaysInline::recursivelyVisitUsers(<br>
+ GlobalValue &GV,<br>
+ SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {<br>
+ SmallVector<User *, 16> Stack;<br>
+<br>
+ SmallPtrSet<const Value *, 8> Visited;<br>
+<br>
+ for (User *U : GV.users())<br>
+ Stack.push_back(U);<br>
+<br>
+ while (!Stack.empty()) {<br>
+ User *U = Stack.pop_back_val();<br>
+ if (!Visited.insert(U).second)<br>
+ continue;<br>
+<br>
+ if (Instruction *I = dyn_cast<Instruction>(U)) {<br>
+ Function *F = I->getParent()->getParent();<br>
+ if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {<br>
+ FuncsToAlwaysInline.insert(F);<br>
+ Stack.push_back(F);<br>
+ }<br>
+<br>
+ // No need to look at further users, but we do need to inline any callers.<br>
+ continue;<br>
+ }<br>
+<br>
+ for (User *UU : U->users())<br>
+ Stack.push_back(UU);<br>
+ }<br>
+}<br>
+<br>
bool AMDGPUAlwaysInline::runOnModule(Module &M) {<br>
+ AMDGPUAS AMDGPUAS = AMDGPU::getAMDGPUAS(M);<br>
+<br>
std::vector<GlobalAlias*> AliasesToRemove;<br>
- std::vector<Function *> FuncsToClone;<br>
+<br>
+ SmallPtrSet<Function *, 8> FuncsToAlwaysInline;<br>
+ SmallPtrSet<Function *, 8> FuncsToNoInline;<br>
<br>
for (GlobalAlias &A : M.aliases()) {<br>
if (Function* F = dyn_cast<Function>(A.getAliasee())) {<br>
A.replaceAllUsesWith(F);<br>
AliasesToRemove.push_back(&A);<br>
}<br>
+<br>
+ // FIXME: If the aliasee isn't a function, it's some kind of constant expr<br>
+ // cast that won't be inlined through.<br>
}<br>
<br>
if (GlobalOpt) {<br>
@@ -63,31 +109,51 @@ bool AMDGPUAlwaysInline::runOnModule(Mod<br>
}<br>
}<br>
<br>
- auto NewAttr = StressCalls ? Attribute::NoInline : Attribute::AlwaysInline;<br>
- auto IncompatAttr<br>
- = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;<br>
-<br>
- for (Function &F : M) {<br>
- if (!F.hasLocalLinkage() && !F.isDeclaration() && !F.use_empty() &&<br>
- !F.hasFnAttribute(IncompatAttr))<br>
- FuncsToClone.push_back(&F);<br>
- }<br>
+ // Always force inlining of any function that uses an LDS global address. This<br>
+ // is something of a workaround because we don't have a way of supporting LDS<br>
+ // objects defined in functions. LDS is always allocated by a kernel, and it<br>
+ // is difficult to manage LDS usage if a function may be used by multiple<br>
+ // kernels.<br>
+ //<br>
+ // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this<br>
+ // should only appear when IPO passes manages to move LDs defined in a kernel<br>
+ // into a single user function.<br>
+<br>
+ for (GlobalVariable &GV : M.globals()) {<br>
+ // TODO: Region address<br>
+ unsigned AS = GV.getType()->getAddressSpace();<br>
+ if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS.REGION_ADDRESS)<br>
+ continue;<br>
<br>
- for (Function *F : FuncsToClone) {<br>
- ValueToValueMapTy VMap;<br>
- Function *NewFunc = CloneFunction(F, VMap);<br>
- NewFunc->setLinkage(GlobalValue::InternalLinkage);<br>
- F->replaceAllUsesWith(NewFunc);<br>
+ recursivelyVisitUsers(GV, FuncsToAlwaysInline);<br>
}<br>
<br>
- for (Function &F : M) {<br>
- if (F.hasLocalLinkage() && !F.hasFnAttribute(IncompatAttr)) {<br>
- F.addFnAttr(NewAttr);<br>
+ if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {<br>
+ auto IncompatAttr<br>
+ = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;<br>
+<br>
+ for (Function &F : M) {<br>
+ if (!F.isDeclaration() && !F.use_empty() &&<br>
+ !F.hasFnAttribute(IncompatAttr)) {<br>
+ if (StressCalls) {<br>
+ if (!FuncsToAlwaysInline.count(&F))<br>
+ FuncsToNoInline.insert(&F);<br>
+ } else<br>
+ FuncsToAlwaysInline.insert(&F);<br>
+ }<br>
}<br>
}<br>
- return false;<br>
+<br>
+ for (Function *F : FuncsToAlwaysInline)<br>
+ F->addFnAttr(Attribute::AlwaysInline);<br>
+<br>
+ for (Function *F : FuncsToNoInline)<br>
+ F->addFnAttr(Attribute::NoInline);<br>
+<br>
+ return !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();<br>
}<br>
<br>
ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {<br>
return new AMDGPUAlwaysInline(GlobalOpt);<br>
}<br>
+<br>
<br>
Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp?rev=336587&r1=336586&r2=336587&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp?rev=336587&r1=336586&r2=336587&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp (original)<br>
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp Mon Jul 9 12:22:22 2018<br>
@@ -117,11 +117,12 @@ static cl::opt<bool, true> LateCFGStruct<br>
cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG),<br>
cl::Hidden);<br>
<br>
-static cl::opt<bool> EnableAMDGPUFunctionCalls(<br>
+static cl::opt<bool, true> EnableAMDGPUFunctionCalls(<br>
"amdgpu-function-calls",<br>
- cl::Hidden,<br>
cl::desc("Enable AMDGPU function call support"),<br>
- cl::init(false));<br>
+ cl::location(AMDGPUTargetMachine::EnableFunctionCalls),<br>
+ cl::init(false),<br>
+ cl::Hidden);<br>
<br>
// Enable lib calls simplifications<br>
static cl::opt<bool> EnableLibCallSimplify(<br>
@@ -311,9 +312,10 @@ AMDGPUTargetMachine::AMDGPUTargetMachine<br>
initAsmInfo();<br>
}<br>
<br>
-AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;<br>
-<br>
bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;<br>
+bool AMDGPUTargetMachine::EnableFunctionCalls = false;<br>
+<br>
+AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;<br>
<br>
StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {<br>
Attribute GPUAttr = F.getFnAttribute("target-cpu");<br>
<br>
Modified: llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h?rev=336587&r1=336586&r2=336587&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h?rev=336587&r1=336586&r2=336587&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h (original)<br>
+++ llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.h Mon Jul 9 12:22:22 2018<br>
@@ -41,6 +41,7 @@ protected:<br>
<br>
public:<br>
static bool EnableLateStructurizeCFG;<br>
+ static bool EnableFunctionCalls;<br>
<br>
AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,<br>
StringRef FS, TargetOptions Options,<br>
<br>
Modified: llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll?rev=336587&r1=336586&r2=336587&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll?rev=336587&r1=336586&r2=336587&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll (original)<br>
+++ llvm/trunk/test/CodeGen/AMDGPU/early-inline.ll Mon Jul 9 12:22:22 2018<br>
@@ -16,10 +16,18 @@ entry:<br>
; CHECK: mul i32<br>
; CHECK-NOT: call i32<br>
<br>
-; CHECK: define i32 @c_alias<br>
define amdgpu_kernel void @caller(i32 %x) {<br>
entry:<br>
%res = call i32 @callee(i32 %x)<br>
store volatile i32 %res, i32 addrspace(1)* undef<br>
ret void<br>
}<br>
+<br>
+; CHECK-LABEL: @alias_caller(<br>
+; CHECK-NOT: call<br>
+define amdgpu_kernel void @alias_caller(i32 %x) {<br>
+entry:<br>
+ %res = call i32 @c_alias(i32 %x)<br>
+ store volatile i32 %res, i32 addrspace(1)* undef<br>
+ ret void<br>
+}<br>
<br>
Added: llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll?rev=336587&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll?rev=336587&view=auto</a><br>
==============================================================================<br>
--- llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll (added)<br>
+++ llvm/trunk/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll Mon Jul 9 12:22:22 2018<br>
@@ -0,0 +1,77 @@<br>
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-function-calls -amdgpu-always-inline %s | FileCheck -check-prefixes=CALLS-ENABLED,ALL %s<br>
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-function-calls -amdgpu-stress-function-calls -amdgpu-always-inline %s | FileCheck -check-prefixes=STRESS-CALLS,ALL %s<br>
+<br>
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5"<br>
+<br>
+@lds0 = addrspace(3) global i32 undef, align 4<br>
+@lds1 = addrspace(3) global [512 x i32] undef, align 4<br>
+@nested.lds.address = addrspace(1) global i32 addrspace(3)* @lds0, align 4<br>
+@gds0 = addrspace(2) global i32 undef, align 4<br>
+<br>
+@alias.lds0 = alias i32, i32 addrspace(3)* @lds0<br>
+@lds.cycle = addrspace(3) global i32 ptrtoint (i32 addrspace(3)* @lds.cycle to i32), align 4<br>
+<br>
+<br>
+; ALL-LABEL: define i32 @load_lds_simple() #0 {<br>
+define i32 @load_lds_simple() {<br>
+ %load = load i32, i32 addrspace(3)* @lds0, align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @load_gds_simple() #0 {<br>
+define i32 @load_gds_simple() {<br>
+ %load = load i32, i32 addrspace(2)* @gds0, align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @load_lds_const_gep() #0 {<br>
+define i32 @load_lds_const_gep() {<br>
+ %load = load i32, i32 addrspace(3)* getelementptr inbounds ([512 x i32], [512 x i32] addrspace(3)* @lds1, i64 0, i64 4), align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @load_lds_var_gep(i32 %idx) #0 {<br>
+define i32 @load_lds_var_gep(i32 %idx) {<br>
+ %gep = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds1, i32 0, i32 %idx<br>
+ %load = load i32, i32 addrspace(3)* %gep, align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 addrspace(3)* @load_nested_address(i32 %idx) #0 {<br>
+define i32 addrspace(3)* @load_nested_address(i32 %idx) {<br>
+ %load = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(1)* @nested.lds.address, align 4<br>
+ ret i32 addrspace(3)* %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @load_lds_alias() #0 {<br>
+define i32 @load_lds_alias() {<br>
+ %load = load i32, i32 addrspace(3)* @alias.lds0, align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @load_lds_cycle() #0 {<br>
+define i32 @load_lds_cycle() {<br>
+ %load = load i32, i32 addrspace(3)* @lds.cycle, align 4<br>
+ ret i32 %load<br>
+}<br>
+<br>
+; ALL-LABEL: define i1 @icmp_lds_address() #0 {<br>
+define i1 @icmp_lds_address() {<br>
+ ret i1 icmp eq (i32 addrspace(3)* @lds0, i32 addrspace(3)* null)<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @transitive_call() #0 {<br>
+define i32 @transitive_call() {<br>
+ %call = call i32 @load_lds_simple()<br>
+ ret i32 %call<br>
+}<br>
+<br>
+; ALL-LABEL: define i32 @recursive_call_lds(i32 %arg0) #0 {<br>
+define i32 @recursive_call_lds(i32 %arg0) {<br>
+ %load = load i32, i32 addrspace(3)* @lds0, align 4<br>
+ %add = add i32 %arg0, %load<br>
+ %call = call i32 @recursive_call_lds(i32 %add)<br>
+ ret i32 %call<br>
+}<br>
+<br>
+; ALL: attributes #0 = { alwaysinline }<br>
<br>
Modified: llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll?rev=336587&r1=336586&r2=336587&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll?rev=336587&r1=336586&r2=336587&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll (original)<br>
+++ llvm/trunk/test/CodeGen/AMDGPU/stress-calls.ll Mon Jul 9 12:22:22 2018<br>
@@ -1,4 +1,4 @@<br>
-; RUN: opt -S -amdgpu-stress-function-calls -amdgpu-always-inline %s | FileCheck %s<br>
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-stress-function-calls -amdgpu-always-inline %s | FileCheck %s<br>
<br>
; CHECK: define internal fastcc i32 @alwaysinline_func(i32 %a) #0 {<br>
define internal fastcc i32 @alwaysinline_func(i32 %a) alwaysinline {<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div></div></div>