[llvm] d6695e1 - [llvm] Add interface to drive inlining decision using ML model
Mircea Trofin via llvm-commits
llvm-commits at lists.llvm.org
Wed May 13 13:28:08 PDT 2020
Author: Mircea Trofin
Date: 2020-05-13T13:27:29-07:00
New Revision: d6695e18763a05b30cb336c18157175277da8f4b
URL: https://github.com/llvm/llvm-project/commit/d6695e18763a05b30cb336c18157175277da8f4b
DIFF: https://github.com/llvm/llvm-project/commit/d6695e18763a05b30cb336c18157175277da8f4b.diff
LOG: [llvm] Add interface to drive inlining decision using ML model
Summary:
This change introduces InliningAdvisor (and related APIs), the interface
that abstracts decision making away from the inlining pass. We will use
this interface to delegate decision making to a trained ML model,
subsequently (see referenced RFC).
RFC: http://lists.llvm.org/pipermail/llvm-dev/2020-April/140763.html
Reviewers: davidxl, eraman, dblaikie
Subscribers: mgorny, hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D79042
Added:
llvm/test/Transforms/Inline/inlining-advisor-default.ll
Modified:
llvm/include/llvm/Analysis/InlineAdvisor.h
llvm/include/llvm/Passes/PassBuilder.h
llvm/include/llvm/Transforms/IPO/Inliner.h
llvm/lib/Analysis/InlineAdvisor.cpp
llvm/lib/Passes/PassBuilder.cpp
llvm/lib/Passes/PassRegistry.def
llvm/lib/Transforms/IPO/Inliner.cpp
llvm/test/Other/new-pm-defaults.ll
llvm/test/Other/new-pm-lto-defaults.ll
llvm/test/Other/new-pm-thinlto-defaults.ll
llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
llvm/test/Other/scc-deleted-printer.ll
llvm/test/Other/scc-pass-printer.ll
llvm/test/Transforms/Inline/inline_stats.ll
llvm/test/Transforms/Inline/internal-scc-members.ll
llvm/test/Transforms/Inline/module-inlining.ll
llvm/test/Transforms/Inline/monster_scc.ll
llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
llvm/test/Transforms/Inline/optimization-remarks.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 3dab78223d0b..9f90c65dff17 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -20,7 +20,178 @@ namespace llvm {
class BasicBlock;
class CallBase;
class Function;
+class Module;
class OptimizationRemarkEmitter;
+class PreservedAnalyses;
+
+/// There are 3 scenarios we can use the InlineAdvisor:
+/// - Default - use manual heuristics.
+///
+/// - Release mode, the expected mode for production, day to day deployments.
+/// In this mode, when building the compiler, we also compile a pre-trained ML
+/// model to native code, and link it as a static library. This mode has low
+/// overhead and no additional dependencies for the compiler runtime.
+///
+/// - Development mode, for training new models.
+/// In this mode, we trade off runtime performance for flexibility. This mode
+/// requires the full C Tensorflow API library, and evaluates models
+/// dynamically. This mode also permits generating training logs, for offline
+/// training.
+enum class InliningAdvisorMode : int { Default, Release, Development };
+
+class InlineAdvisor;
+/// Capture state between an inlining decision having had been made, and
+/// its impact being observable. When collecting model training data, this
+/// allows recording features/decisions/partial reward data sets.
+///
+/// Derivations of this type are expected to be tightly coupled with their
+/// InliningAdvisors. The base type implements the minimal contractual
+/// obligations.
+class InlineAdvice {
+public:
+ InlineAdvice(InlineAdvice &&) = delete;
+ InlineAdvice(const InlineAdvice &) = delete;
+ virtual ~InlineAdvice() {
+ assert(Recorded && "InlineAdvice should have been informed of the "
+ "inliner's decision in all cases");
+ }
+
+ /// Exactly one of the record* APIs must be called. Implementers may extend
+ /// behavior by implementing the corresponding record*Impl.
+ ///
+ /// Call after inlining succeeded, and did not result in deleting the callee.
+ void recordInlining() {
+ markRecorded();
+ recordInliningImpl();
+ }
+
+ /// Call after inlining succeeded, and resulted in deleting the callee.
+ void recordInliningWithCalleeDeleted();
+
+ /// Call after the decision for a call site was to not inline.
+ void recordUnsuccessfulInlining(const InlineResult &Result) {
+ markRecorded();
+ recordUnsuccessfulInliningImpl(Result);
+ }
+
+ /// Call to indicate inlining was not attempted.
+ void recordUnattemptedInlining() {
+ markRecorded();
+ recordUnattemptedInliningImpl();
+ }
+
+ /// Get the inlining recommendation.
+ bool isInliningRecommended() const { return IsInliningRecommended; }
+
+protected:
+ InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+ bool IsInliningRecommended);
+
+ virtual void recordInliningImpl() {}
+ virtual void recordInliningWithCalleeDeletedImpl() {}
+ virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {}
+ virtual void recordUnattemptedInliningImpl() {}
+
+ InlineAdvisor *const Advisor;
+ /// Caller and Callee are pre-inlining.
+ Function *const Caller;
+ Function *const Callee;
+ const bool IsInliningRecommended;
+
+private:
+ void markRecorded() {
+ assert(!Recorded && "Recording should happen exactly once");
+ Recorded = true;
+ }
+
+ bool Recorded = false;
+};
+
+/// Interface for deciding whether to inline a call site or not.
+class InlineAdvisor {
+public:
+ InlineAdvisor(InlineAdvisor &&) = delete;
+ virtual ~InlineAdvisor() { freeDeletedFunctions(); }
+
+ /// Get an InlineAdvice containing a recommendation on whether to
+ /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to
+ /// be up-to-date wrt previous inlining decisions.
+ /// Returns an InlineAdvice with the inlining recommendation.
+ virtual std::unique_ptr<InlineAdvice>
+ getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) = 0;
+
+ /// This must be called when the Inliner pass is entered, to allow the
+ /// InlineAdvisor update internal state, as result of function passes run
+ /// between Inliner pass runs (for the same module).
+ virtual void OnPassEntry() {}
+
+ /// This must be called when the Inliner pass is exited, as function passes
+ /// may be run subsequently. This allows an implementation of InlineAdvisor
+ /// to prepare for a partial update.
+ virtual void OnPassExit() {}
+
+protected:
+ InlineAdvisor() = default;
+
+ /// We may want to defer deleting functions to after the inlining for a whole
+ /// module has finished. This allows us to reliably use function pointers as
+ /// unique identifiers, as an efficient implementation detail of the
+ /// InlineAdvisor. Otherwise, it is possible the memory allocator
+ /// re-allocate Function objects at the same address of a deleted Function;
+ /// and Functions are potentially created during the function passes called
+ /// after each SCC inlining (e.g. argument promotion does that).
+ void freeDeletedFunctions();
+
+ bool isFunctionDeleted(Function *F) const {
+ return DeletedFunctions.count(F);
+ }
+
+private:
+ friend class InlineAdvice;
+ void markFunctionAsDeleted(Function *F);
+ std::unordered_set<Function *> DeletedFunctions;
+};
+
+/// The default (manual heuristics) implementation of the InlineAdvisor. This
+/// implementation does not need to keep state between inliner pass runs, and is
+/// reusable as-is for inliner pass test scenarios, as well as for regular use.
+class DefaultInlineAdvisor : public InlineAdvisor {
+public:
+ DefaultInlineAdvisor(InlineParams Params) : Params(Params) {}
+
+private:
+ std::unique_ptr<InlineAdvice>
+ getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) override;
+
+ void OnPassExit() override { freeDeletedFunctions(); }
+ InlineParams Params;
+};
+
+/// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor
+/// needs to capture state right before inlining commences over a module.
+class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
+public:
+ static AnalysisKey Key;
+ InlineAdvisorAnalysis() = default;
+ struct Result {
+ Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {}
+ bool invalidate(Module &, const PreservedAnalyses &,
+ ModuleAnalysisManager::Invalidator &) {
+ // InlineAdvisor must be preserved across analysis invalidations.
+ return false;
+ }
+ bool tryCreate(InlineParams Params, InliningAdvisorMode Mode);
+ InlineAdvisor *getAdvisor() const { return Advisor.get(); }
+ void clear() { Advisor.reset(); }
+
+ private:
+ Module &M;
+ ModuleAnalysisManager &MAM;
+ std::unique_ptr<InlineAdvisor> Advisor;
+ };
+
+ Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
+};
// Default (manual policy) decision making helper APIs. Shared with the legacy
// pass manager inliner.
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index a68c9db12b83..391d144d5dcd 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -19,6 +19,7 @@
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Error.h"
+#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include <vector>
@@ -345,9 +346,9 @@ class PassBuilder {
/// Construct the module pipeline that performs inlining as well as
/// the inlining-driven cleanups.
- ModulePassManager buildInlinerPipeline(OptimizationLevel Level,
- ThinLTOPhase Phase,
- bool DebugLogging = false);
+ ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level,
+ ThinLTOPhase Phase,
+ bool DebugLogging = false);
/// Construct the core LLVM module optimization pipeline.
///
diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h
index 026ad3a693a1..442c12a9adcf 100644
--- a/llvm/include/llvm/Transforms/IPO/Inliner.h
+++ b/llvm/include/llvm/Transforms/IPO/Inliner.h
@@ -11,6 +11,7 @@
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/PassManager.h"
@@ -93,21 +94,54 @@ struct LegacyInlinerBase : public CallGraphSCCPass {
/// passes be composed to achieve the same end result.
class InlinerPass : public PassInfoMixin<InlinerPass> {
public:
- InlinerPass(InlineParams Params = getInlineParams())
- : Params(std::move(Params)) {}
+ InlinerPass() = default;
~InlinerPass();
InlinerPass(InlinerPass &&Arg)
- : Params(std::move(Arg.Params)),
- ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
+ : ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
LazyCallGraph &CG, CGSCCUpdateResult &UR);
private:
- InlineParams Params;
+ InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+ Module &M);
std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
+ Optional<DefaultInlineAdvisor> OwnedDefaultAdvisor;
};
+/// Module pass, wrapping the inliner pass. This works in conjunction with the
+/// InlineAdvisorAnalysis to facilitate inlining decisions taking into account
+/// module-wide state, that need to keep track of inter-inliner pass runs, for
+/// a given module. An InlineAdvisor is configured and kept alive for the
+/// duration of the ModuleInlinerWrapperPass::run.
+class ModuleInlinerWrapperPass
+ : public PassInfoMixin<ModuleInlinerWrapperPass> {
+public:
+ ModuleInlinerWrapperPass(
+ InlineParams Params = getInlineParams(), bool Debugging = false,
+ InliningAdvisorMode Mode = InliningAdvisorMode::Default,
+ unsigned MaxDevirtIterations = 0);
+ ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
+
+ PreservedAnalyses run(Module &, ModuleAnalysisManager &);
+
+ /// Allow adding more CGSCC passes, besides inlining. This should be called
+ /// before run is called, as part of pass pipeline building.
+ CGSCCPassManager &getPM() { return PM; }
+
+ /// Allow adding module-level analyses benefiting the contained CGSCC passes.
+ template <class T> void addRequiredModuleAnalysis() {
+ MPM.addPass(RequireAnalysisPass<T, Module>());
+ }
+
+private:
+ const InlineParams Params;
+ const InliningAdvisorMode Mode;
+ const unsigned MaxDevirtIterations;
+ const bool Debugging;
+ CGSCCPassManager PM;
+ ModulePassManager MPM;
+};
} // end namespace llvm
#endif // LLVM_TRANSFORMS_IPO_INLINER_H
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index e9d4f7e16ddc..2cc57d0f750b 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -7,7 +7,8 @@
//
//===----------------------------------------------------------------------===//
//
-// This file implements inlining decision-making APIs.
+// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
+// related types.
//
//===----------------------------------------------------------------------===//
@@ -46,6 +47,126 @@ static cl::opt<int>
cl::desc("Scale to limit the cost of inline deferral"),
cl::init(-1), cl::Hidden);
+namespace {
+class DefaultInlineAdvice : public InlineAdvice {
+public:
+ DefaultInlineAdvice(DefaultInlineAdvisor *Advisor, CallBase &CB,
+ Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE)
+ : InlineAdvice(Advisor, CB, OIC.hasValue()), OriginalCB(&CB), OIC(OIC),
+ ORE(ORE), DLoc(CB.getDebugLoc()), Block(CB.getParent()) {}
+
+private:
+ void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
+ using namespace ore;
+ llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
+ "; " + inlineCostStr(*OIC));
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
+ << NV("Callee", Callee) << " will not be inlined into "
+ << NV("Caller", Caller) << ": "
+ << NV("Reason", Result.getFailureReason());
+ });
+ }
+
+ void recordInliningWithCalleeDeletedImpl() override {
+ emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+ }
+
+ void recordInliningImpl() override {
+ emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+ }
+
+private:
+ CallBase *const OriginalCB;
+ Optional<InlineCost> OIC;
+ OptimizationRemarkEmitter &ORE;
+
+ // Capture the context of CB before inlining, as a successful inlining may
+ // change that context, and we want to report success or failure in the
+ // original context.
+ const DebugLoc DLoc;
+ const BasicBlock *const Block;
+};
+
+} // namespace
+
+std::unique_ptr<InlineAdvice>
+DefaultInlineAdvisor::getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) {
+ Function &Callee = *CB.getCalledFunction();
+ Function &F = *CB.getCaller();
+ ProfileSummaryInfo *PSI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+ .getCachedResult<ProfileSummaryAnalysis>(
+ *CB.getParent()->getParent()->getParent());
+
+ auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ // FIXME: make GetAssumptionCache's decl similar to the other 2 below. May
+ // need changing the type of getInlineCost parameters? Also see similar case
+ // in Inliner.cpp
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache =
+ [&](Function &F) -> AssumptionCache & {
+ return FAM.getResult<AssumptionAnalysis>(Callee);
+ };
+ auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
+ return FAM.getResult<BlockFrequencyAnalysis>(F);
+ };
+ auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+ return FAM.getResult<TargetLibraryAnalysis>(F);
+ };
+
+ auto GetInlineCost = [&](CallBase &CB) {
+ Function &Callee = *CB.getCalledFunction();
+ auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
+ bool RemarksEnabled =
+ Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
+ DEBUG_TYPE);
+ return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
+ GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
+ };
+ auto OIC = llvm::shouldInline(CB, GetInlineCost, ORE);
+ return std::make_unique<DefaultInlineAdvice>(this, CB, OIC, ORE);
+}
+
+InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+ bool IsInliningRecommended)
+ : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
+ IsInliningRecommended(IsInliningRecommended) {}
+
+void InlineAdvisor::markFunctionAsDeleted(Function *F) {
+ assert((!DeletedFunctions.count(F)) &&
+ "Cannot put cause a function to become dead twice!");
+ DeletedFunctions.insert(F);
+}
+
+void InlineAdvisor::freeDeletedFunctions() {
+ for (auto *F : DeletedFunctions)
+ delete F;
+ DeletedFunctions.clear();
+}
+
+void InlineAdvice::recordInliningWithCalleeDeleted() {
+ markRecorded();
+ Advisor->markFunctionAsDeleted(Callee);
+ recordInliningWithCalleeDeletedImpl();
+}
+
+AnalysisKey InlineAdvisorAnalysis::Key;
+
+bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
+ InliningAdvisorMode Mode) {
+ switch (Mode) {
+ case InliningAdvisorMode::Default:
+ Advisor.reset(new DefaultInlineAdvisor(Params));
+ break;
+ case InliningAdvisorMode::Development:
+ // To be added subsequently under conditional compilation.
+ break;
+ case InliningAdvisorMode::Release:
+ // To be added subsequently under conditional compilation.
+ break;
+ }
+ return !!Advisor;
+}
+
/// Return true if inlining of CB can block the caller from being
/// inlined which is proved to be more beneficial. \p IC is the
/// estimated inline cost associated with callsite \p CB.
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index b098f7777261..bdde94abaa7a 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -33,6 +33,7 @@
#include "llvm/Analysis/DominanceFrontier.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVUsers.h"
+#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/LazyValueInfo.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
@@ -215,6 +216,16 @@ static cl::opt<bool> EnableGVNHoist(
"enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
+static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
+ "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
+ cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
+ cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
+ "Heuristics-based inliner version."),
+ clEnumValN(InliningAdvisorMode::Development, "development",
+ "Use development mode (runtime-loadable model)."),
+ clEnumValN(InliningAdvisorMode::Release, "release",
+ "Use release mode (AOT-compiled model).")));
+
static cl::opt<bool> EnableGVNSink(
"enable-npm-gvn-sink", cl::init(false), cl::Hidden,
cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
@@ -614,10 +625,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
// This should probably be lowered after performance testing.
// FIXME: this comment is cargo culted from the old pass manager, revisit).
IP.HintThreshold = 325;
-
- CGSCCPassManager CGPipeline(DebugLogging);
-
- CGPipeline.addPass(InlinerPass(IP));
+ ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
+ CGSCCPassManager &CGPipeline = MIWP.getPM();
FunctionPassManager FPM;
FPM.addPass(SROA());
@@ -628,7 +637,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
- MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
+ MPM.addPass(std::move(MIWP));
// Delete anything that is now dead to make sure that we don't instrument
// dead code. Instrumentation can end up keeping dead code around and
@@ -693,41 +702,36 @@ getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
}
-ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
- ThinLTOPhase Phase,
- bool DebugLogging) {
- ModulePassManager MPM(DebugLogging);
+ModuleInlinerWrapperPass
+PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
+ bool DebugLogging) {
+ InlineParams IP = getInlineParamsFromOptLevel(Level);
+ if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ IP.HotCallSiteThreshold = 0;
+
+ ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
+ MaxDevirtIterations);
// Require the GlobalsAA analysis for the module so we can query it within
// the CGSCC pipeline.
- MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
+ MIWP.addRequiredModuleAnalysis<GlobalsAA>();
// Require the ProfileSummaryAnalysis for the module so we can query it within
// the inliner pass.
- MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
// Now begin the main postorder CGSCC pipeline.
// FIXME: The current CGSCC pipeline has its origins in the legacy pass
// manager and trying to emulate its precise behavior. Much of this doesn't
// make a lot of sense and we should revisit the core CGSCC structure.
- CGSCCPassManager MainCGPipeline(DebugLogging);
+ CGSCCPassManager &MainCGPipeline = MIWP.getPM();
// Note: historically, the PruneEH pass was run first to deduce nounwind and
// generally clean up exception handling overhead. It isn't clear this is
// valuable as the inliner doesn't currently care whether it is inlining an
// invoke or a call.
- // Run the inliner first. The theory is that we are walking bottom-up and so
- // the callees have already been fully optimized, and we want to inline them
- // into the callers so that our optimizations can reflect that.
- // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
- // because it makes profile annotation in the backend inaccurate.
- InlineParams IP = getInlineParamsFromOptLevel(Level);
- if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
- PGOOpt->Action == PGOOptions::SampleUse)
- IP.HotCallSiteThreshold = 0;
- MainCGPipeline.addPass(InlinerPass(IP));
-
if (AttributorRun & AttributorRunOption::CGSCC)
MainCGPipeline.addPass(AttributorCGSCCPass());
@@ -755,15 +759,7 @@ ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
for (auto &C : CGSCCOptimizerLateEPCallbacks)
C(MainCGPipeline, Level);
- // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
- // to detect when we devirtualize indirect calls and iterate the SCC passes
- // in that case to try and catch knock-on inlining or function attrs
- // opportunities. Then we add it to the module pipeline by walking the SCCs
- // in postorder (or bottom-up).
- MPM.addPass(
- createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
- std::move(MainCGPipeline), MaxDevirtIterations)));
- return MPM;
+ return MIWP;
}
ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
@@ -1330,8 +1326,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
// valuable as the inliner doesn't currently care whether it is inlining an
// invoke or a call.
// Run the inliner now.
- MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
- InlinerPass(getInlineParamsFromOptLevel(Level))));
+ MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
+ DebugLogging));
// Optimize globals again after we ran the inliner.
MPM.addPass(GlobalOptPass());
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 3566332960c0..ec33f8aab796 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -27,6 +27,7 @@ MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis())
MODULE_ANALYSIS("verify", VerifierAnalysis())
MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
MODULE_ANALYSIS("asan-globals-md", ASanGlobalsMetadataAnalysis())
+MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
#ifndef MODULE_ALIAS_ANALYSIS
#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
@@ -57,6 +58,7 @@ MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false))
MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true))
MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
MODULE_PASS("instrorderfile", InstrOrderFilePass())
MODULE_PASS("instrprof", InstrProfiling())
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index e18039e7d0a1..7ff905912257 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -17,6 +17,7 @@
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
@@ -28,6 +29,7 @@
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LazyCallGraph.h"
@@ -664,6 +666,24 @@ InlinerPass::~InlinerPass() {
}
}
+InlineAdvisor &
+InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+ Module &M) {
+ auto *IAA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
+ if (!IAA) {
+ // It should still be possible to run the inliner as a stand-alone SCC pass,
+ // for test scenarios. In that case, we default to the
+ // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
+ // runs. It also uses just the default InlineParams.
+ OwnedDefaultAdvisor.emplace(getInlineParams());
+ return *OwnedDefaultAdvisor;
+ }
+ assert(IAA->getAdvisor() &&
+ "Expected a present InlineAdvisorAnalysis also have an "
+ "InlineAdvisor initialized");
+ return *IAA->getAdvisor();
+}
+
PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
CGSCCAnalysisManager &AM, LazyCallGraph &CG,
CGSCCUpdateResult &UR) {
@@ -675,6 +695,11 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
Module &M = *InitialC.begin()->getFunction().getParent();
ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
+ InlineAdvisor &Advisor = getAdvisor(MAMProxy, M);
+ Advisor.OnPassEntry();
+
+ auto AdvisorOnExit = make_scope_exit([&] { Advisor.OnPassExit(); });
+
if (!ImportedFunctionsStats &&
InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
ImportedFunctionsStats =
@@ -779,29 +804,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n");
- // Get the remarks emission analysis for the caller.
- auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-
std::function<AssumptionCache &(Function &)> GetAssumptionCache =
[&](Function &F) -> AssumptionCache & {
return FAM.getResult<AssumptionAnalysis>(F);
};
- auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
- return FAM.getResult<BlockFrequencyAnalysis>(F);
- };
- auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
- return FAM.getResult<TargetLibraryAnalysis>(F);
- };
-
- auto GetInlineCost = [&](CallBase &CB) {
- Function &Callee = *CB.getCalledFunction();
- auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
- bool RemarksEnabled =
- Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
- DEBUG_TYPE);
- return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
- GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
- };
// Now process as many calls as we have within this caller in the sequnece.
// We bail out as soon as the caller has to change so we can update the
@@ -833,101 +839,88 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
continue;
}
- auto OIC = shouldInline(*CB, GetInlineCost, ORE);
+ auto Advice = Advisor.getAdvice(*CB, FAM);
// Check whether we want to inline this callsite.
- if (!OIC)
+ if (!Advice->isInliningRecommended()) {
+ Advice->recordUnattemptedInlining();
continue;
- auto DoInline = [&]() -> InlineResult {
- // Setup the data structure used to plumb customization into the
- // `InlineFunction` routine.
- InlineFunctionInfo IFI(
- /*cg=*/nullptr, &GetAssumptionCache, PSI,
- &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
- &FAM.getResult<BlockFrequencyAnalysis>(Callee));
-
- InlineResult IR = InlineFunction(*CB, IFI);
- if (!IR.isSuccess())
- return IR;
-
- DidInline = true;
- InlinedCallees.insert(&Callee);
- ++NumInlined;
+ }
- // Add any new callsites to defined functions to the worklist.
- if (!IFI.InlinedCallSites.empty()) {
- int NewHistoryID = InlineHistory.size();
- InlineHistory.push_back({&Callee, InlineHistoryID});
-
- for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
- Function *NewCallee = ICB->getCalledFunction();
- if (!NewCallee) {
- // Try to promote an indirect (virtual) call without waiting for
- // the post-inline cleanup and the next DevirtSCCRepeatedPass
- // iteration because the next iteration may not happen and we may
- // miss inlining it.
- if (tryPromoteCall(*ICB))
- NewCallee = ICB->getCalledFunction();
- }
- if (NewCallee)
- if (!NewCallee->isDeclaration())
- Calls.push_back({ICB, NewHistoryID});
- }
- }
+ // Setup the data structure used to plumb customization into the
+ // `InlineFunction` routine.
+ InlineFunctionInfo IFI(
+ /*cg=*/nullptr, &GetAssumptionCache, PSI,
+ &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
+ &FAM.getResult<BlockFrequencyAnalysis>(Callee));
- if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
- ImportedFunctionsStats->recordInline(F, Callee);
-
- // Merge the attributes based on the inlining.
- AttributeFuncs::mergeAttributesForInlining(F, Callee);
-
- // For local functions, check whether this makes the callee trivially
- // dead. In that case, we can drop the body of the function eagerly
- // which may reduce the number of callers of other functions to one,
- // changing inline cost thresholds.
- if (Callee.hasLocalLinkage()) {
- // To check this we also need to nuke any dead constant uses (perhaps
- // made dead by this operation on other functions).
- Callee.removeDeadConstantUsers();
- if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
- Calls.erase(
- std::remove_if(Calls.begin() + I + 1, Calls.end(),
- [&](const std::pair<CallBase *, int> &Call) {
- return Call.first->getCaller() == &Callee;
- }),
- Calls.end());
- // Clear the body and queue the function itself for deletion when we
- // finish inlining and call graph updates.
- // Note that after this point, it is an error to do anything other
- // than use the callee's address or delete it.
- Callee.dropAllReferences();
- assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
- "Cannot put cause a function to become dead twice!");
- DeadFunctions.push_back(&Callee);
+ InlineResult IR = InlineFunction(*CB, IFI);
+ if (!IR.isSuccess()) {
+ Advice->recordUnsuccessfulInlining(IR);
+ continue;
+ }
+
+ DidInline = true;
+ InlinedCallees.insert(&Callee);
+ ++NumInlined;
+
+ // Add any new callsites to defined functions to the worklist.
+ if (!IFI.InlinedCallSites.empty()) {
+ int NewHistoryID = InlineHistory.size();
+ InlineHistory.push_back({&Callee, InlineHistoryID});
+
+ for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
+ Function *NewCallee = ICB->getCalledFunction();
+ if (!NewCallee) {
+ // Try to promote an indirect (virtual) call without waiting for
+ // the post-inline cleanup and the next DevirtSCCRepeatedPass
+ // iteration because the next iteration may not happen and we may
+ // miss inlining it.
+ if (tryPromoteCall(*ICB))
+ NewCallee = ICB->getCalledFunction();
}
+ if (NewCallee)
+ if (!NewCallee->isDeclaration())
+ Calls.push_back({ICB, NewHistoryID});
}
- return IR;
- };
- // Capture the context of CB before inlining, as a successful inlining may
- // change that context, and we want to report success or failure in the
- // original context.
- auto DLoc = CB->getDebugLoc();
- auto *Block = CB->getParent();
-
- auto Outcome = DoInline();
- if (!Outcome.isSuccess()) {
- using namespace ore;
- setInlineRemark(*CB, std::string(Outcome.getFailureReason()) + "; " +
- inlineCostStr(*OIC));
- ORE.emit([&]() {
- return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
- << NV("Callee", &Callee) << " will not be inlined into "
- << NV("Caller", &F) << ": "
- << NV("Reason", Outcome.getFailureReason());
- });
- continue;
}
- emitInlinedInto(ORE, DLoc, Block, Callee, F, *OIC);
+ if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
+ ImportedFunctionsStats->recordInline(F, Callee);
+
+ // Merge the attributes based on the inlining.
+ AttributeFuncs::mergeAttributesForInlining(F, Callee);
+
+ // For local functions, check whether this makes the callee trivially
+ // dead. In that case, we can drop the body of the function eagerly
+ // which may reduce the number of callers of other functions to one,
+ // changing inline cost thresholds.
+ bool CalleeWasDeleted = false;
+ if (Callee.hasLocalLinkage()) {
+ // To check this we also need to nuke any dead constant uses (perhaps
+ // made dead by this operation on other functions).
+ Callee.removeDeadConstantUsers();
+ if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
+ Calls.erase(
+ std::remove_if(Calls.begin() + I + 1, Calls.end(),
+ [&](const std::pair<CallBase *, int> &Call) {
+ return Call.first->getCaller() == &Callee;
+ }),
+ Calls.end());
+ // Clear the body and queue the function itself for deletion when we
+ // finish inlining and call graph updates.
+ // Note that after this point, it is an error to do anything other
+ // than use the callee's address or delete it.
+ Callee.dropAllReferences();
+ assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
+ "Cannot put cause a function to become dead twice!");
+ DeadFunctions.push_back(&Callee);
+ CalleeWasDeleted = true;
+ }
+ }
+ if (CalleeWasDeleted)
+ Advice->recordInliningWithCalleeDeleted();
+ else
+ Advice->recordInlining();
}
// Back the call index up by one to put us in a good position to go around
@@ -1008,7 +1001,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
// sets.
for (Function *DeadF : DeadFunctions) {
// Get the necessary information out of the call graph and nuke the
- // function there. Also, cclear out any cached analyses.
+ // function there. Also, clear out any cached analyses.
auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
FAM.clear(*DeadF, DeadF->getName());
AM.clear(DeadC, DeadC.getName());
@@ -1021,7 +1014,15 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
UR.InvalidatedRefSCCs.insert(&DeadRC);
// And delete the actual function from the module.
- M.getFunctionList().erase(DeadF);
+ // The Advisor may use Function pointers to efficiently index various
+ // internal maps, e.g. for memoization. Function cleanup passes like
+ // argument promotion create new functions. It is possible for a new
+ // function to be allocated at the address of a deleted function. We could
+ // index using names, but that's inefficient. Alternatively, we let the
+ // Advisor free the functions when it sees fit.
+ DeadF->getBasicBlockList().clear();
+ M.getFunctionList().remove(DeadF);
+
++NumDeleted;
}
@@ -1034,3 +1035,45 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
return PA;
}
+
+ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params,
+ bool Debugging,
+ InliningAdvisorMode Mode,
+ unsigned MaxDevirtIterations)
+ : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations),
+ Debugging(Debugging), PM(Debugging), MPM(Debugging) {
+ // Run the inliner first. The theory is that we are walking bottom-up and so
+ // the callees have already been fully optimized, and we want to inline them
+ // into the callers so that our optimizations can reflect that.
+ // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
+ // because it makes profile annotation in the backend inaccurate.
+ PM.addPass(InlinerPass());
+}
+
+PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
+ ModuleAnalysisManager &MAM) {
+ auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
+ if (!IAA.tryCreate(Params, Mode)) {
+ M.getContext().emitError(
+ "Could not setup Inlining Advisor for the requested "
+ "mode and/or options");
+ return PreservedAnalyses::all();
+ }
+
+ // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
+ // to detect when we devirtualize indirect calls and iterate the SCC passes
+ // in that case to try and catch knock-on inlining or function attrs
+ // opportunities. Then we add it to the module pipeline by walking the SCCs
+ // in postorder (or bottom-up).
+ // If MaxDevirtIterations is 0, we just don't use the devirtualization
+ // wrapper.
+ if (MaxDevirtIterations == 0)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(PM)));
+ else
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createDevirtSCCRepeatedPass(std::move(PM), MaxDevirtIterations)));
+ auto Ret = MPM.run(M, MAM);
+
+ IAA.clear();
+ return Ret;
+}
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 9aa02654d9e9..ae3e1af86762 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -127,7 +127,8 @@
; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting llvm::Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 4bfee73720f7..29263f633a8d 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -73,7 +73,14 @@
; CHECK-O2-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
; CHECK-O2-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}InlinerPass>
+; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-O2-NEXT: Starting CGSCC pass manager run.
+; CHECK-O2-NEXT: Running pass: InlinerPass
+; CHECK-O2-NEXT: Finished CGSCC pass manager run.
+; CHECK-O2-NEXT: Finished llvm::Module pass manager run.
; CHECK-O2-NEXT: Running pass: GlobalOptPass
; CHECK-O2-NEXT: Running pass: GlobalDCEPass
; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index 73a14b1b1300..6bbad8941ea1 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -92,7 +92,8 @@
; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting llvm::Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index 205a93630b70..c84a74d1ffac 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -66,7 +66,8 @@
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index 5ac657ef909d..d4895bad34ef 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -75,7 +75,8 @@
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index d43cad7ab29c..0baf7df0a3d7 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -1,7 +1,7 @@
; Validate ThinLTO prelink pipeline when we have instrumentation PGO
;
; RUN: llvm-profdata merge %S/Inputs/new-pm-thinlto-prelink-pgo-defaults.proftext -o %t.profdata
-;
+;
; RUN: opt -disable-verify -debug-pass-manager \
; RUN: -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
; RUN: -passes='thinlto-pre-link<O1>,name-anon-globals' -S %s 2>&1 \
@@ -65,6 +65,9 @@
; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
+; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O123-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager<{{.*}}LazyCallGraph::SCC
; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
@@ -75,6 +78,7 @@
; CHECK-O123-NEXT: Running pass: InlinerPass on (foo)
; CHECK-O123-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
; CHECK-O123-NEXT: Finished CGSCC pass manager run.
+; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
; CHECK-O123-NEXT: Running pass: GlobalDCEPass
; CHECK-O-NEXT: Running pass: PGOInstrumentationUse
; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
@@ -93,7 +97,9 @@
; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo
; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis on foo
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index 87f47b78cb48..439b553903c9 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -75,7 +75,8 @@
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
diff --git a/llvm/test/Other/scc-deleted-printer.ll b/llvm/test/Other/scc-deleted-printer.ll
index f7b97c3d30eb..188322b999d9 100644
--- a/llvm/test/Other/scc-deleted-printer.ll
+++ b/llvm/test/Other/scc-deleted-printer.ll
@@ -3,6 +3,11 @@
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-before-all -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+
; INL: IR Dump Before {{InlinerPass .*scc: .tester, foo}}
; INL-NOT: IR Dump After {{InlinerPass}}
; INL: IR Dump Before {{InlinerPass .*scc: .tester}}
diff --git a/llvm/test/Other/scc-pass-printer.ll b/llvm/test/Other/scc-pass-printer.ll
index 9e12a285f025..9f0625d57218 100644
--- a/llvm/test/Other/scc-pass-printer.ll
+++ b/llvm/test/Other/scc-pass-printer.ll
@@ -3,9 +3,13 @@
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-after-all | FileCheck %s -check-prefix=INL
; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
; RUN: -inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
; INL: IR Dump After {{Function Integration/Inlining|InlinerPass .*scc: .bar, foo}}
; INL: define void @bar()
diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll
index 40d6cb30b695..2fe43925fb98 100644
--- a/llvm/test/Transforms/Inline/inline_stats.ll
+++ b/llvm/test/Transforms/Inline/inline_stats.ll
@@ -6,6 +6,9 @@
; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+
; CHECK: ------- Dumping inliner stats for [<stdin>] -------
; CHECK-BASIC-NOT: -- List of inlined functions:
; CHECK-BASIC-NOT: -- Inlined not imported function
diff --git a/llvm/test/Transforms/Inline/inlining-advisor-default.ll b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
new file mode 100644
index 000000000000..808ddd97dc6b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
@@ -0,0 +1,9 @@
+; Check that, in the absence of dependencies, we emit an error message when
+; trying to use ML-driven inlining.
+;
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -S < %s 2>&1 | FileCheck %s
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %s 2>&1 | FileCheck %s
+
+declare i64 @f1()
+
+; CHECK: Could not setup Inlining Advisor for the requested mode and/or options
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/internal-scc-members.ll b/llvm/test/Transforms/Inline/internal-scc-members.ll
index 258ce00744c5..c12ff6cfdc58 100644
--- a/llvm/test/Transforms/Inline/internal-scc-members.ll
+++ b/llvm/test/Transforms/Inline/internal-scc-members.ll
@@ -3,6 +3,7 @@
;
; RUN: opt < %s -S -inline | FileCheck %s
; RUN: opt < %s -S -passes=inline | FileCheck %s
+; RUN: opt < %s -S -passes=inliner-wrapper | FileCheck %s
; CHECK-LABEL: define internal void @test1_scc0()
; CHECK-NOT: call
diff --git a/llvm/test/Transforms/Inline/module-inlining.ll b/llvm/test/Transforms/Inline/module-inlining.ll
index 5fdd07f0c085..00bfabd4982d 100644
--- a/llvm/test/Transforms/Inline/module-inlining.ll
+++ b/llvm/test/Transforms/Inline/module-inlining.ll
@@ -6,6 +6,7 @@
; a 'ret 10'
;
; RUN: opt -passes=inline -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
+; RUN: opt -passes=inliner-wrapper -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
; RUN: opt -passes=scc-oz-module-inliner -S < %s | FileCheck %s --check-prefix=MODULE --check-prefix=CHECK
define void @modify_value({i32, float}* %v) {
diff --git a/llvm/test/Transforms/Inline/monster_scc.ll b/llvm/test/Transforms/Inline/monster_scc.ll
index b32a2aed331e..860e2e4fb225 100644
--- a/llvm/test/Transforms/Inline/monster_scc.ll
+++ b/llvm/test/Transforms/Inline/monster_scc.ll
@@ -41,6 +41,7 @@
;
; RUN: opt -S < %s -inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,OLD
; RUN: opt -S < %s -passes=inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
+; RUN: opt -S < %s -passes=inliner-wrapper -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
index 14b413b74375..9299b2c6a29a 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
@@ -5,6 +5,10 @@
; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
; RUN: FileCheck -allow-empty -check-prefix=THRESHOLD %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
+; RUN: FileCheck -allow-empty -check-prefix=THRESHOLD %s
+
; Check that when any threshold is specified we ignore remarks with no
; hotness -- these are blocks that have not been executed during training.
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
index 3d572219a351..df0cd9b78d67 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
@@ -8,6 +8,11 @@
; RUN: -pass-remarks-with-hotness 2>&1 | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN: -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN: -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
; Check the YAML file for inliner-generated passed and analysis remarks. This
; is the input:
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
index 1a1c0f4bac12..6457ed32cdec 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
@@ -4,6 +4,9 @@
; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
; RUN: | FileCheck %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
+; RUN: | FileCheck %s
; CHECK: foo inlined into bar with (cost=always): always inline attribute (hotness: 30)
; CHECK: foz not inlined into bar because it should never be inlined (cost=never): noinline function attribute (hotness: 30)
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
index 10a93f5cd79a..d1b44d847f1b 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
@@ -34,6 +34,25 @@
; RUN: opt < %s -S -passes=inline \
; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
; RUN: -pass-remarks-output=%t.threshold
+
+; Inliner - Module Wrapper
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 15 \
+; RUN: -pass-remarks-output=%t 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-with-hotness -pass-remarks-output=%t
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+;
+; Verify that remarks that don't meet the hotness threshold are not output.
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN: -pass-remarks-output=%t.threshold 2>&1 | \
+; RUN: FileCheck -check-prefix=THRESHOLD %s
+; RUN: test ! -s %t.threshold
+; RUN: opt < %s -S -passes=inliner-wrapper \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN: -pass-remarks-output=%t.threshold
+
; The remarks output file should be empty.
; RUN: test ! -s %t.threshold
diff --git a/llvm/test/Transforms/Inline/optimization-remarks.ll b/llvm/test/Transforms/Inline/optimization-remarks.ll
index a3a21a848da8..1a15849a6823 100644
--- a/llvm/test/Transforms/Inline/optimization-remarks.ll
+++ b/llvm/test/Transforms/Inline/optimization-remarks.ll
@@ -12,6 +12,13 @@
; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
; RUN: FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -S 2>&1 | \
+; RUN: FileCheck -check-prefix=CHECK -check-prefix=NO_HOTNESS %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
+; RUN: FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+
; HOTNESS: fox will not be inlined into bar because its definition is unavailable
; NO_HOTNESS-NOT: fox will not be inlined into bar because its definition is unavailable
; CHECK: foo inlined into bar with (cost=always): always inline attribute
More information about the llvm-commits
mailing list