[llvm] b11391b - ThinLTO : Import always_inline functions irrespective of the threshold
Teresa Johnson via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 8 17:03:53 PST 2019
Author: Teresa Johnson
Date: 2019-11-08T17:02:01-08:00
New Revision: b11391bb47d6fb75639c331378440b405e64be7a
URL: https://github.com/llvm/llvm-project/commit/b11391bb47d6fb75639c331378440b405e64be7a
DIFF: https://github.com/llvm/llvm-project/commit/b11391bb47d6fb75639c331378440b405e64be7a.diff
LOG: ThinLTO : Import always_inline functions irrespective of the threshold
Summary: A user can force a function to be inlined by specifying the always_inline attribute. Currently, thinlto implementation is not aware of always_inline functions and does not guarantee import of such functions, which in turn can prevent inlining of such functions.
Patch by Bharathi Seshadri <bseshadr at cisco.com>
Reviewers: tejohnson
Reviewed By: tejohnson
Subscribers: mehdi_amini, inglorion, hiraditya, steven_wu, dexonsmith, arphaman, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D70014
Added:
llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
Modified:
llvm/include/llvm/IR/ModuleSummaryIndex.h
llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
llvm/lib/AsmParser/LLLexer.cpp
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/AsmParser/LLToken.h
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/IR/AsmWriter.cpp
llvm/lib/IR/ModuleSummaryIndex.cpp
llvm/lib/Transforms/IPO/FunctionImport.cpp
llvm/test/Assembler/thinlto-summary.ll
llvm/test/ThinLTO/X86/dot-dumper.ll
llvm/test/ThinLTO/X86/dot-dumper2.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index ae611b774478..210641da651a 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -547,6 +547,8 @@ class FunctionSummary : public GlobalValueSummary {
// Indicate if the global value cannot be inlined.
unsigned NoInline : 1;
+ // Indicate if function should be always inlined.
+ unsigned AlwaysInline : 1;
};
/// Create an empty FunctionSummary (with specified call edges).
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 8232bf07cafc..0f0f27ecdb41 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -466,7 +466,8 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
// FIXME: refactor this to use the same code that inliner is using.
// Don't try to import functions with noinline attribute.
- F.getAttributes().hasFnAttribute(Attribute::NoInline)};
+ F.getAttributes().hasFnAttribute(Attribute::NoInline),
+ F.hasFnAttribute(Attribute::AlwaysInline)};
auto FuncSummary = std::make_unique<FunctionSummary>(
Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
CallGraphEdges.takeVector(), TypeTests.takeVector(),
@@ -703,7 +704,8 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
F->hasFnAttribute(Attribute::ReadOnly),
F->hasFnAttribute(Attribute::NoRecurse),
F->returnDoesNotAlias(),
- /* NoInline = */ false},
+ /* NoInline = */ false,
+ F->hasFnAttribute(Attribute::AlwaysInline)},
/*EntryCount=*/0, ArrayRef<ValueInfo>{},
ArrayRef<FunctionSummary::EdgeTy>{},
ArrayRef<GlobalValue::GUID>{},
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index c6df821a22cd..887a931c64d2 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -750,6 +750,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(noRecurse);
KEYWORD(returnDoesNotAlias);
KEYWORD(noInline);
+ KEYWORD(alwaysInline);
KEYWORD(calls);
KEYWORD(callee);
KEYWORD(hotness);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index c204a9448ce0..e9d2dfc195b1 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -8248,6 +8248,8 @@ bool LLParser::ParseFlag(unsigned &Val) {
/// [',' 'readOnly' ':' Flag]? [',' 'noRecurse' ':' Flag]?
/// [',' 'returnDoesNotAlias' ':' Flag]? ')'
/// [',' 'noInline' ':' Flag]? ')'
+/// [',' 'alwaysInline' ':' Flag]? ')'
+
bool LLParser::ParseOptionalFFlags(FunctionSummary::FFlags &FFlags) {
assert(Lex.getKind() == lltok::kw_funcFlags);
Lex.Lex();
@@ -8289,6 +8291,12 @@ bool LLParser::ParseOptionalFFlags(FunctionSummary::FFlags &FFlags) {
return true;
FFlags.NoInline = Val;
break;
+ case lltok::kw_alwaysInline:
+ Lex.Lex();
+ if (ParseToken(lltok::colon, "expected ':'") || ParseFlag(Val))
+ return true;
+ FFlags.AlwaysInline = Val;
+ break;
default:
return Error(Lex.getLoc(), "expected function flag type");
}
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
index e418f7cbfb49..9153b49aa045 100644
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -382,6 +382,7 @@ enum Kind {
kw_noRecurse,
kw_returnDoesNotAlias,
kw_noInline,
+ kw_alwaysInline,
kw_calls,
kw_callee,
kw_hotness,
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index b65c088ac92f..92c3e1d2dd8f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -960,6 +960,7 @@ static FunctionSummary::FFlags getDecodedFFlags(uint64_t RawFlags) {
Flags.NoRecurse = (RawFlags >> 2) & 0x1;
Flags.ReturnDoesNotAlias = (RawFlags >> 3) & 0x1;
Flags.NoInline = (RawFlags >> 4) & 0x1;
+ Flags.AlwaysInline = (RawFlags >> 5) & 0x1;
return Flags;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 97f5bc9ee45d..c3531a7ecace 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -1006,6 +1006,7 @@ static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
RawFlags |= (Flags.NoRecurse << 2);
RawFlags |= (Flags.ReturnDoesNotAlias << 3);
RawFlags |= (Flags.NoInline << 4);
+ RawFlags |= (Flags.AlwaysInline << 5);
return RawFlags;
}
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 9ae15f49da80..107b32ea3263 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -2959,6 +2959,7 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) {
Out << ", noRecurse: " << FFlags.NoRecurse;
Out << ", returnDoesNotAlias: " << FFlags.ReturnDoesNotAlias;
Out << ", noInline: " << FFlags.NoInline;
+ Out << ", alwaysInline: " << FFlags.AlwaysInline;
Out << ")";
}
if (!FS->calls().empty()) {
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index b575b5b64428..518027aba5b5 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -330,7 +330,7 @@ static std::string fflagsToString(FunctionSummary::FFlags F) {
auto FlagValue = [](unsigned V) { return V ? '1' : '0'; };
char FlagRep[] = {FlagValue(F.ReadNone), FlagValue(F.ReadOnly),
FlagValue(F.NoRecurse), FlagValue(F.ReturnDoesNotAlias),
- FlagValue(F.NoInline), 0};
+ FlagValue(F.NoInline), FlagValue(F.AlwaysInline), 0};
return FlagRep;
}
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 83b03a07c457..4f2a7001e0f2 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -231,7 +231,8 @@ selectCallee(const ModuleSummaryIndex &Index,
return false;
}
- if (Summary->instCount() > Threshold) {
+ if ((Summary->instCount() > Threshold) &&
+ !Summary->fflags().AlwaysInline) {
Reason = FunctionImporter::ImportFailureReason::TooLarge;
return false;
}
@@ -475,7 +476,8 @@ static void computeImportForFunction(
CalleeSummary = CalleeSummary->getBaseObject();
ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
- assert(ResolvedCalleeSummary->instCount() <= NewThreshold &&
+ assert((ResolvedCalleeSummary->fflags().AlwaysInline ||
+ (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&
"selectCallee() didn't honor the threshold");
auto ExportModulePath = ResolvedCalleeSummary->modulePath();
diff --git a/llvm/test/Assembler/thinlto-summary.ll b/llvm/test/Assembler/thinlto-summary.ll
index d4be32f015cf..dea535cc47f1 100644
--- a/llvm/test/Assembler/thinlto-summary.ll
+++ b/llvm/test/Assembler/thinlto-summary.ll
@@ -38,7 +38,7 @@
; Functions with various flag combinations (notEligibleToImport, Live,
; combinations of optional function flags).
^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0), insts: 1, funcFlags: (noInline: 1))))
-^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readNone: 1, noRecurse: 1))))
+^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readNone: 1, noRecurse: 1, alwaysInline: 1))))
; This one also tests backwards reference in calls.
^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0), insts: 1, funcFlags: (readOnly: 1, returnDoesNotAlias: 1), calls: ((callee: ^15)))))
@@ -80,9 +80,9 @@
; CHECK: ^12 = gv: (guid: 11, summaries: (variable: (module: ^0, flags: (linkage: appending, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), varFlags: (readonly: 0, writeonly: 0), refs: (^4))))
; CHECK: ^13 = gv: (guid: 12, summaries: (variable: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), varFlags: (readonly: 1, writeonly: 0))))
; CHECK: ^14 = gv: (guid: 13, summaries: (variable: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), varFlags: (readonly: 0, writeonly: 0))))
-; CHECK: ^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1))))
-; CHECK: ^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0))))
-; CHECK: ^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 1, noRecurse: 0, returnDoesNotAlias: 1, noInline: 0), calls: ((callee: ^15)))))
+; CHECK: ^15 = gv: (guid: 14, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 1, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0))))
+; CHECK: ^16 = gv: (guid: 15, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 1))))
+; CHECK: ^17 = gv: (guid: 16, summaries: (function: (module: ^1, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 1, funcFlags: (readNone: 0, readOnly: 1, noRecurse: 0, returnDoesNotAlias: 1, noInline: 0, alwaysInline: 0), calls: ((callee: ^15)))))
; CHECK: ^18 = gv: (guid: 17, summaries: (alias: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), aliasee: ^14)))
; CHECK: ^19 = gv: (guid: 18, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, typeIdInfo: (typeTests: (^24, ^26)))))
; CHECK: ^20 = gv: (guid: 19, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 8, typeIdInfo: (typeTestAssumeVCalls: (vFuncId: (^27, offset: 16))))))
diff --git a/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll b/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
new file mode 100644
index 000000000000..31ca7f5b3f65
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/funcimport_alwaysinline.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+ call void (...) @foo()
+ ret i32 0
+}
+
+declare void @foo(...)
diff --git a/llvm/test/ThinLTO/X86/dot-dumper.ll b/llvm/test/ThinLTO/X86/dot-dumper.ll
index 92925426d425..d2a6c9dbd70c 100644
--- a/llvm/test/ThinLTO/X86/dot-dumper.ll
+++ b/llvm/test/ThinLTO/X86/dot-dumper.ll
@@ -21,7 +21,7 @@
; PERMODULE-NEXT: label = "";
; PERMODULE-NEXT: node [style=filled,fillcolor=lightblue];
; PERMODULE-NEXT: M0_[[MAIN_ALIAS:[0-9]+]] [style="dotted,filled",shape="box",label="main_alias",fillcolor="red"]; // alias, dead
-; PERMODULE-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 00000)}",fillcolor="red"]; // function, dead
+; PERMODULE-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 000000)}",fillcolor="red"]; // function, dead
; PERMODULE-NEXT: // Edges:
; PERMODULE-NEXT: M0_[[MAIN_ALIAS]] -> M0_[[MAIN]] [style=dotted]; // alias
; PERMODULE-NEXT: }
@@ -40,7 +40,7 @@
; COMBINED-NEXT: label = "dot-dumper{{.*}}1.bc";
; COMBINED-NEXT: node [style=filled,fillcolor=lightblue];
; COMBINED-NEXT: M0_[[MAIN_ALIAS:[0-9]+]] [style="dotted,filled",shape="box",label="main_alias",fillcolor="red"]; // alias, dead
-; COMBINED-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 00000)}"]; // function
+; COMBINED-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 4, ffl: 000000)}"]; // function
; COMBINED-NEXT: // Edges:
; COMBINED-NEXT: M0_[[MAIN_ALIAS]] -> M0_[[MAIN]] [style=dotted]; // alias
; COMBINED-NEXT: }
@@ -50,10 +50,10 @@
; COMBINED-NEXT: color = lightgrey;
; COMBINED-NEXT: label = "dot-dumper{{.*}}2.bc";
; COMBINED-NEXT: node [style=filled,fillcolor=lightblue];
-; COMBINED-NEXT: M1_[[FOO:[0-9]+]] [shape="record",label="foo|extern (inst: 4, ffl: 00001)}"]; // function
+; COMBINED-NEXT: M1_[[FOO:[0-9]+]] [shape="record",label="foo|extern (inst: 4, ffl: 000010)}"]; // function
; COMBINED-NEXT: M1_[[A:[0-9]+]] [shape="Mrecord",label="A|extern}"]; // variable, immutable
; COMBINED-NEXT: M1_[[B:[0-9]+]] [shape="Mrecord",label="B|extern}"]; // variable, immutable
-; COMBINED-NEXT: M1_{{[0-9]+}} [shape="record",label="bar|extern (inst: 1, ffl: 00000)}",fillcolor="red"]; // function, dead
+; COMBINED-NEXT: M1_{{[0-9]+}} [shape="record",label="bar|extern (inst: 1, ffl: 000000)}",fillcolor="red"]; // function, dead
; COMBINED-NEXT: // Edges:
; COMBINED-NEXT: M1_[[FOO]] -> M1_[[B]] [style=dashed,color=forestgreen]; // const-ref
; COMBINED-NEXT: M1_[[FOO]] -> M1_[[A]] [style=dashed,color=forestgreen]; // const-ref
diff --git a/llvm/test/ThinLTO/X86/dot-dumper2.ll b/llvm/test/ThinLTO/X86/dot-dumper2.ll
index 1bfa4e0e6e71..95bfa020f6dc 100644
--- a/llvm/test/ThinLTO/X86/dot-dumper2.ll
+++ b/llvm/test/ThinLTO/X86/dot-dumper2.ll
@@ -15,7 +15,7 @@
; COMBINED-NEXT: color = lightgrey;
; COMBINED-NEXT: label =
; COMBINED-NEXT: node [style=filled,fillcolor=lightblue];
-; COMBINED-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 2, ffl: 00000)}"]; // function
+; COMBINED-NEXT: M0_[[MAIN:[0-9]+]] [shape="record",label="main|extern (inst: 2, ffl: 000000)}"]; // function
; COMBINED-NEXT: // Edges:
; COMBINED-NEXT: }
; COMBINED-NEXT: // Module:
diff --git a/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll b/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
new file mode 100644
index 000000000000..38e63b25af17
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/funcimport_alwaysinline.ll
@@ -0,0 +1,24 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_alwaysinline.ll -o %t2.bc
+
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
+; RUN: -r=%t1.bc,foo,plx \
+; RUN: -r=%t2.bc,main,plx \
+; RUN: -r=%t2.bc,foo,l \
+; RUN: -import-instr-limit=0
+; RUN: llvm-dis %t.o.2.3.import.bc -o - | FileCheck %s
+
+; foo() being always_inline should be imported irrespective of the
+; instruction limit
+; CHECK: define available_externally dso_local void @foo()
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: alwaysinline nounwind uwtable
+define void @foo() #0 {
+entry:
+ ret void
+}
+
+attributes #0 = { alwaysinline nounwind uwtable }
More information about the llvm-commits
mailing list