[llvm] r293657 - Explicitly promote indirect calls before sample profile annotation.
Dehao Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 31 09:49:37 PST 2017
Author: dehao
Date: Tue Jan 31 11:49:37 2017
New Revision: 293657
URL: http://llvm.org/viewvc/llvm-project?rev=293657&view=rev
Log:
Explicitly promote indirect calls before sample profile annotation.
Summary: In iterative sample pgo where profile is collected from PGOed binary, we may see indirect call targets promoted and inlined in the profile. Before profile annotation, we need to make this happen in order to annotate correctly on IR. This patch explicitly promotes these indirect calls and inlines them before profile annotation.
Reviewers: xur, davidxl
Reviewed By: davidxl
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D29040
Modified:
llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp
llvm/trunk/test/Transforms/SampleProfile/Inputs/indirect-call.prof
llvm/trunk/test/Transforms/SampleProfile/indirect-call.ll
Modified: llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp?rev=293657&r1=293656&r2=293657&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp (original)
+++ llvm/trunk/lib/Transforms/IPO/SampleProfile.cpp Tue Jan 31 11:49:37 2017
@@ -52,6 +52,7 @@
#include "llvm/Support/Format.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include <cctype>
@@ -614,14 +615,14 @@ SampleProfileLoader::findFunctionSamples
/// Iteratively traverse all callsites of the function \p F, and find if
/// the corresponding inlined instance exists and is hot in profile. If
/// it is hot enough, inline the callsites and adds new callsites of the
-/// callee into the caller.
-///
-/// TODO: investigate the possibility of not invoking InlineFunction directly.
+/// callee into the caller. If the call is an indirect call, first promote
+/// it to direct call. Each indirect call is limited with a single target.
///
/// \param F function to perform iterative inlining.
///
/// \returns True if there is any inline happened.
bool SampleProfileLoader::inlineHotFunctions(Function &F) {
+ DenseSet<Instruction *> PromotedInsns;
bool Changed = false;
LLVMContext &Ctx = F.getContext();
std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
@@ -647,10 +648,28 @@ bool SampleProfileLoader::inlineHotFunct
}
for (auto I : CIS) {
InlineFunctionInfo IFI(nullptr, ACT ? &GetAssumptionCache : nullptr);
- CallSite CS(I);
- Function *CalledFunction = CS.getCalledFunction();
+ Function *CalledFunction = CallSite(I).getCalledFunction();
+ Instruction *DI = I;
+ if (!CalledFunction && !PromotedInsns.count(I)) {
+ auto CalleeFunctionName = findCalleeFunctionSamples(*I)->getName();
+ const char *Reason = "Callee function not available";
+ CalledFunction = F.getParent()->getFunction(CalleeFunctionName);
+ if (CalledFunction && isLegalToPromote(I, CalledFunction, &Reason)) {
+ // The indirect target was promoted and inlined in the profile, as a
+ // result, we do not have profile info for the branch probability.
+ // We set the probability to 80% taken to indicate that the static
+ // call is likely taken.
+ DI = promoteIndirectCall(I, CalledFunction, 80, 100);
+ PromotedInsns.insert(I);
+ } else {
+ DEBUG(dbgs() << "\nFailed to promote indirect call to "
+ << CalleeFunctionName << " because " << Reason << "\n");
+ continue;
+ }
+ }
if (!CalledFunction || !CalledFunction->getSubprogram())
continue;
+ CallSite CS(DI);
DebugLoc DLoc = I->getDebugLoc();
uint64_t NumSamples = findCalleeFunctionSamples(*I)->getTotalSamples();
if (InlineFunction(CS, IFI)) {
Modified: llvm/trunk/test/Transforms/SampleProfile/Inputs/indirect-call.prof
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SampleProfile/Inputs/indirect-call.prof?rev=293657&r1=293656&r2=293657&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/SampleProfile/Inputs/indirect-call.prof (original)
+++ llvm/trunk/test/Transforms/SampleProfile/Inputs/indirect-call.prof Tue Jan 31 11:49:37 2017
@@ -1,2 +1,8 @@
test:63067:0
4: 3345 _Z3barv:1398 _Z3foov:2059
+test_inline:3000:0
+ 5: foo_inline:3000
+ 1: 3000
+test_noinline:3000:0
+ 5: foo_noinline:3000
+ 1: 3000
Modified: llvm/trunk/test/Transforms/SampleProfile/indirect-call.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SampleProfile/indirect-call.ll?rev=293657&r1=293656&r2=293657&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/SampleProfile/indirect-call.ll (original)
+++ llvm/trunk/test/Transforms/SampleProfile/indirect-call.ll Tue Jan 31 11:49:37 2017
@@ -1,5 +1,6 @@
; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.prof -S | FileCheck %s
+; CHECK-LABEL: @test
define void @test(void ()*) !dbg !3 {
%2 = alloca void ()*
store void ()* %0, void ()** %2
@@ -9,6 +10,41 @@ define void @test(void ()*) !dbg !3 {
ret void
}
+; CHECK-LABEL: @test_inline
+; If the indirect call is promoted and inlined in profile, we should promote and inline it.
+define void @test_inline(void ()*) !dbg !3 {
+ %2 = alloca void ()*
+ store void ()* %0, void ()** %2
+ %3 = load void ()*, void ()** %2
+; CHECK: icmp {{.*}} @foo_inline
+; CHECK: if.true.direct_targ:
+; CHECK-NOT: call
+; CHECK: if.false.orig_indirect:
+; CHECK: call
+ call void %3(), !dbg !5
+ ret void
+}
+
+; CHECK-LABEL: @test_noinline
+; If the indirect call target is not available, we should not promote it.
+define void @test_noinline(void ()*) !dbg !3 {
+ %2 = alloca void ()*
+ store void ()* %0, void ()** %2
+ %3 = load void ()*, void ()** %2
+; CHECK-NOT: icmp
+; CHECK: call
+ call void %3(), !dbg !5
+ ret void
+}
+
+define void @foo_inline() !dbg !3 {
+ ret void
+}
+
+define i32 @foo_noinline(i32 %x) !dbg !3 {
+ ret i32 %x
+}
+
!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!2}
@@ -17,4 +53,5 @@ define void @test(void ()*) !dbg !3 {
!2 = !{i32 2, !"Debug Info Version", i32 3}
!3 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, unit: !0)
!4 = !DILocation(line: 5, scope: !3)
+!5 = !DILocation(line: 6, scope: !3)
; CHECK: ![[PROF]] = !{!"VP", i32 0, i64 3457, i64 9191153033785521275, i64 2059, i64 -1069303473483922844, i64 1398}
More information about the llvm-commits
mailing list