[llvm] e34e27f - [TTI][AMDGPU] Allow targets to adjust `LastCallToStaticBonus` via `getInliningLastCallToStaticBonus` (#111311)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 11 07:19:57 PDT 2024
Author: Shilei Tian
Date: 2024-10-11T10:19:54-04:00
New Revision: e34e27f19820af958db7c3b93de7f489aa3bf4dc
URL: https://github.com/llvm/llvm-project/commit/e34e27f19820af958db7c3b93de7f489aa3bf4dc
DIFF: https://github.com/llvm/llvm-project/commit/e34e27f19820af958db7c3b93de7f489aa3bf4dc.diff
LOG: [TTI][AMDGPU] Allow targets to adjust `LastCallToStaticBonus` via `getInliningLastCallToStaticBonus` (#111311)
Currently we will not be able to inline a large function even if it only
has one live use because the inline cost is still very high after
applying `LastCallToStaticBonus`, which is a constant. This could
significantly impact the performance because CSR spill is very
expensive.
This PR adds a new function `getInliningLastCallToStaticBonus` to TTI to
allow targets to customize this value.
Fixes SWDEV-471398.
Added:
llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll
Modified:
llvm/include/llvm/Analysis/InlineAdvisor.h
llvm/include/llvm/Analysis/InlineCost.h
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/lib/Analysis/InlineAdvisor.cpp
llvm/lib/Analysis/InlineCost.cpp
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 871a6e97861e29..700c3b0f18b8d5 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -371,7 +371,8 @@ getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM,
/// using that cost, so we won't do so from this function. Return std::nullopt
/// if inlining should not be attempted.
std::optional<InlineCost>
-shouldInline(CallBase &CB, function_ref<InlineCost(CallBase &CB)> GetInlineCost,
+shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
+ function_ref<InlineCost(CallBase &CB)> GetInlineCost,
OptimizationRemarkEmitter &ORE, bool EnableDeferral = true);
/// Emit ORE message.
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index c5978ce54fc18b..22e0b1bc901a43 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -47,7 +47,6 @@ const int OptAggressiveThreshold = 250;
int getInstrCost();
const int IndirectCallThreshold = 100;
const int LoopPenalty = 25;
-const int LastCallToStaticBonus = 15000;
const int ColdccPenalty = 2000;
/// Do not inline functions which allocate this many bytes on the stack
/// when the caller is recursive.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 5c5da5e06c1bff..64dc9aacd5c57b 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -352,6 +352,9 @@ class TargetTransformInfo {
unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const;
unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const;
+ /// \returns The bonus of inlining the last call to a static function.
+ int getInliningLastCallToStaticBonus() const;
+
/// \returns A value to be added to the inlining threshold.
unsigned adjustInliningThreshold(const CallBase *CB) const;
@@ -1840,6 +1843,7 @@ class TargetTransformInfo::Concept {
virtual unsigned getInliningCostBenefitAnalysisSavingsMultiplier() const = 0;
virtual unsigned
getInliningCostBenefitAnalysisProfitableMultiplier() const = 0;
+ virtual int getInliningLastCallToStaticBonus() const = 0;
virtual unsigned adjustInliningThreshold(const CallBase *CB) = 0;
virtual int getInlinerVectorBonusPercent() const = 0;
virtual unsigned getCallerAllocaCost(const CallBase *CB,
@@ -2250,6 +2254,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const override {
return Impl.getInliningCostBenefitAnalysisProfitableMultiplier();
}
+ int getInliningLastCallToStaticBonus() const override {
+ return Impl.getInliningLastCallToStaticBonus();
+ }
int getInlinerVectorBonusPercent() const override {
return Impl.getInlinerVectorBonusPercent();
}
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 3d0140ad7ad7a3..1c4fcb57755ecf 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -74,6 +74,11 @@ class TargetTransformInfoImplBase {
unsigned getInliningCostBenefitAnalysisProfitableMultiplier() const {
return 8;
}
+ int getInliningLastCallToStaticBonus() const {
+ // This is the value of InlineConstants::LastCallToStaticBonus before it was
+ // removed along with the introduction of this function.
+ return 15000;
+ }
unsigned adjustInliningThreshold(const CallBase *CB) const { return 0; }
unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
return 0;
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index c6907cb128bb47..45702fa25d8b14 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -151,9 +151,9 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
return FAM.getResult<TargetLibraryAnalysis>(F);
};
+ Function &Callee = *CB.getCalledFunction();
+ auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
auto GetInlineCost = [&](CallBase &CB) {
- Function &Callee = *CB.getCalledFunction();
- auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
bool RemarksEnabled =
Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
DEBUG_TYPE);
@@ -161,7 +161,7 @@ std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
};
return llvm::shouldInline(
- CB, GetInlineCost, ORE,
+ CB, CalleeTTI, GetInlineCost, ORE,
Params.EnableDeferral.value_or(EnableInlineDeferral));
}
@@ -247,7 +247,8 @@ bool InlineAdvisorAnalysis::Result::tryCreate(
/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
/// caller if \p CB is suppressed for inlining.
static bool
-shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
+shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI,
+ InlineCost IC, int &TotalSecondaryCost,
function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
// For now we only handle local or inline functions.
if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
@@ -320,7 +321,7 @@ shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
// be removed entirely. We did not account for this above unless there
// is only one caller of Caller.
if (ApplyLastCallBonus)
- TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
+ TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus();
// If InlineDeferralScale is negative, then ignore the cost of primary
// inlining -- IC.getCost() multiplied by the number of callers to Caller.
@@ -374,7 +375,7 @@ void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
/// using that cost, so we won't do so from this function. Return std::nullopt
/// if inlining should not be attempted.
std::optional<InlineCost>
-llvm::shouldInline(CallBase &CB,
+llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
function_ref<InlineCost(CallBase &CB)> GetInlineCost,
OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
using namespace ore;
@@ -413,8 +414,8 @@ llvm::shouldInline(CallBase &CB,
}
int TotalSecondaryCost = 0;
- if (EnableDeferral &&
- shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
+ if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC,
+ TotalSecondaryCost, GetInlineCost)) {
LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
<< " Cost = " << IC.getCost()
<< ", outer Cost = " << TotalSecondaryCost << '\n');
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index d2c329ba748e58..dd9ac910456ade 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1943,7 +1943,7 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) {
// and the callsite.
int SingleBBBonusPercent = 50;
int VectorBonusPercent = TTI.getInlinerVectorBonusPercent();
- int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus;
+ int LastCallToStaticBonus = TTI.getInliningLastCallToStaticBonus();
// Lambda to set all the above bonus and bonus percentages to 0.
auto DisallowAllBonuses = [&]() {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 3dc29fc7cd77b1..8ab8a53b753112 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -228,6 +228,10 @@ TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
}
+int TargetTransformInfo::getInliningLastCallToStaticBonus() const {
+ return TTIImpl->getInliningLastCallToStaticBonus();
+}
+
unsigned
TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
return TTIImpl->adjustInliningThreshold(CB);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 5d7ca89571b27b..8f9495d83cde2d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1314,6 +1314,11 @@ static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,
return AllocaSize;
}
+int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
+ return BaseT::getInliningLastCallToStaticBonus() *
+ getInliningThresholdMultiplier();
+}
+
unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 30da002376251c..022af501289af8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -243,6 +243,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
bool areInlineCompatible(const Function *Caller,
const Function *Callee) const;
+ int getInliningLastCallToStaticBonus() const;
unsigned getInliningThresholdMultiplier() const { return 11; }
unsigned adjustInliningThreshold(const CallBase *CB) const;
unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const;
diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll
new file mode 100644
index 00000000000000..043b73e28b8937
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-only-one-live-use.ll
@@ -0,0 +1,22 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline -inline-threshold=0 -debug-only=inline-cost %s -o - 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: Analyzing call of callee_not_only_one_live_use... (caller:caller)
+; CHECK: Cost: -30
+; CHECK: Analyzing call of callee_only_one_live_use... (caller:caller)
+; CHECK: Cost: -165030
+
+define internal void @callee_not_only_one_live_use() {
+ ret void
+}
+
+define internal void @callee_only_one_live_use() {
+ ret void
+}
+
+define void @caller() {
+ call void @callee_not_only_one_live_use()
+ call void @callee_not_only_one_live_use()
+ call void @callee_only_one_live_use()
+ ret void
+}
More information about the llvm-commits
mailing list