[llvm] [Inline] Accumulate the cost of the inlined function to the new call site (PR #111104)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 4 00:21:21 PDT 2024
https://github.com/DianQK created https://github.com/llvm/llvm-project/pull/111104
Fixes #111102.
Consider the following IR, the `call void @bar` is a local host call site:
```llvm
@define void foo() {
...
loop:
call void @bar()
...
}
define void @bar() {
...
call void @baz()
...
}
define void @baz() {
...
}
```
If `foo` can inline `baz` after inlined `bar`, I think it can also inline a `bar` that has inlined `baz`. With this in mind, I accumulated the previous cost into the new call site, which is a linearly increasing limit.
I did some experimenting on [DianQK/perf/inline-extra-cost](https://llvm-compile-time-tracker.com/?config=Overview&stat=instructions%3Au&remote=DianQK) and the current thresholds look OK. I'm looking forward to getting some feedback before modifying this threshold to an option.
>From 558f121259136d91ca758fe5acd61038f215acb6 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Tue, 1 Oct 2024 19:46:28 +0800
Subject: [PATCH 1/8] [Inline] Accumulate the cost of the inlined function to
the new call site
---
llvm/include/llvm/Analysis/InlineAdvisor.h | 12 +++++++++---
llvm/include/llvm/Analysis/InlineCost.h | 3 +++
llvm/lib/Analysis/InlineAdvisor.cpp | 6 ++++--
llvm/lib/Analysis/InlineCost.cpp | 5 +++++
llvm/lib/Transforms/IPO/Inliner.cpp | 7 +++++++
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 11 ++++++++++-
.../Transforms/Inline/inline-history-noinline.ll | 2 +-
7 files changed, 39 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 871a6e97861e29..2880553817ebd8 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -74,7 +74,8 @@ class InlineAdvisor;
class InlineAdvice {
public:
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
- OptimizationRemarkEmitter &ORE, bool IsInliningRecommended);
+ OptimizationRemarkEmitter &ORE, bool IsInliningRecommended,
+ std::optional<int> InliningCost = std::nullopt);
InlineAdvice(InlineAdvice &&) = delete;
InlineAdvice(const InlineAdvice &) = delete;
@@ -108,6 +109,7 @@ class InlineAdvice {
/// Get the inlining recommendation.
bool isInliningRecommended() const { return IsInliningRecommended; }
+ std::optional<int> inliningCost() const { return InliningCost; }
const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; }
const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; }
@@ -129,6 +131,7 @@ class InlineAdvice {
const BasicBlock *const Block;
OptimizationRemarkEmitter &ORE;
const bool IsInliningRecommended;
+ const std::optional<int> InliningCost;
private:
void markRecorded() {
@@ -145,8 +148,11 @@ class DefaultInlineAdvice : public InlineAdvice {
DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
std::optional<InlineCost> OIC,
OptimizationRemarkEmitter &ORE, bool EmitRemarks = true)
- : InlineAdvice(Advisor, CB, ORE, OIC.has_value()), OriginalCB(&CB),
- OIC(OIC), EmitRemarks(EmitRemarks) {}
+ : InlineAdvice(Advisor, CB, ORE, OIC.has_value(),
+ OIC && OIC->isVariable()
+ ? std::optional<int>(OIC->getCost())
+ : std::nullopt),
+ OriginalCB(&CB), OIC(OIC), EmitRemarks(EmitRemarks) {}
private:
void recordUnsuccessfulInliningImpl(const InlineResult &Result) override;
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index c5978ce54fc18b..44b824305b3f7b 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -59,6 +59,9 @@ const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536;
const char FunctionInlineCostMultiplierAttributeName[] =
"function-inline-cost-multiplier";
+const char FunctionInlineExtraCostAttributeName[] =
+ "function-inline-extra-cost";
+
const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize";
} // namespace InlineConstants
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index c6907cb128bb47..9706dd212d7bc5 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -175,10 +175,12 @@ DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
OptimizationRemarkEmitter &ORE,
- bool IsInliningRecommended)
+ bool IsInliningRecommended,
+ std::optional<int> InliningCost)
: Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
- IsInliningRecommended(IsInliningRecommended) {}
+ IsInliningRecommended(IsInliningRecommended), InliningCost(InliningCost) {
+}
void InlineAdvice::recordInlineStatsIfNeeded() {
if (Advisor->ImportedFunctionsStats)
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index d2c329ba748e58..c379d6a08609bf 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1017,6 +1017,11 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
InlineConstants::FunctionInlineCostMultiplierAttributeName))
Cost *= *AttrCostMult;
+ if (std::optional<int> AttrExtraCost = getStringFnAttrAsInt(
+ CandidateCall,
+ InlineConstants::FunctionInlineExtraCostAttributeName))
+ Cost += *AttrExtraCost;
+
if (std::optional<int> AttrThreshold =
getStringFnAttrAsInt(CandidateCall, "function-inline-threshold"))
Threshold = *AttrThreshold;
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 23ee23eb047f58..746cc34b70b16d 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,6 +376,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
getStringFnAttrAsInt(
*CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
.value_or(1);
+ std::optional<int> InliningCost = Advice->inliningCost();
// Setup the data structure used to plumb customization into the
// `InlineFunction` routine.
@@ -435,6 +436,12 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
InlineConstants::FunctionInlineCostMultiplierAttributeName,
itostr(CBCostMult * IntraSCCCostMultiplier));
ICB->addFnAttr(NewCBCostMult);
+ } else if (InliningCost) {
+ Attribute NewCBExtraCost = Attribute::get(
+ M.getContext(),
+ InlineConstants::FunctionInlineExtraCostAttributeName,
+ itostr(*InliningCost));
+ ICB->addFnAttr(NewCBExtraCost);
}
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index dbc733826944b9..71937ff1172c07 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,6 +225,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
Advice->recordUnattemptedInlining();
continue;
}
+ std::optional<int> InliningCost = Advice->inliningCost();
// Setup the data structure used to plumb customization into the
// `InlineFunction` routine.
@@ -265,8 +266,16 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
NewCallee = ICB->getCalledFunction();
}
if (NewCallee)
- if (!NewCallee->isDeclaration())
+ if (!NewCallee->isDeclaration()) {
Calls->push({ICB, NewHistoryID});
+ if (InliningCost) {
+ Attribute NewCBExtraCost = Attribute::get(
+ M.getContext(),
+ InlineConstants::FunctionInlineExtraCostAttributeName,
+ itostr(*InliningCost));
+ ICB->addFnAttr(NewCBExtraCost);
+ }
+ }
}
}
diff --git a/llvm/test/Transforms/Inline/inline-history-noinline.ll b/llvm/test/Transforms/Inline/inline-history-noinline.ll
index 742bd25ecd9bb9..fbe633fc3c797b 100644
--- a/llvm/test/Transforms/Inline/inline-history-noinline.ll
+++ b/llvm/test/Transforms/Inline/inline-history-noinline.ll
@@ -29,4 +29,4 @@ define internal void @a() {
ret void
}
-; CHECK: [[NOINLINE]] = { noinline }
+; CHECK: [[NOINLINE]] = { noinline {{.*}}}
>From d133af8bb0adc2f8bf4a44b20ba6ad594988efd2 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 19:21:29 +0800
Subject: [PATCH 2/8] Only apply when cost is positive
---
llvm/lib/Transforms/IPO/Inliner.cpp | 2 +-
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 746cc34b70b16d..760d8e21d37e8e 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -436,7 +436,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
InlineConstants::FunctionInlineCostMultiplierAttributeName,
itostr(CBCostMult * IntraSCCCostMultiplier));
ICB->addFnAttr(NewCBCostMult);
- } else if (InliningCost) {
+ } else if (InliningCost && *InliningCost > 0) {
Attribute NewCBExtraCost = Attribute::get(
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 71937ff1172c07..cee31e30e94d46 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -268,7 +268,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
if (NewCallee)
if (!NewCallee->isDeclaration()) {
Calls->push({ICB, NewHistoryID});
- if (InliningCost) {
+ if (InliningCost && *InliningCost > 0) {
Attribute NewCBExtraCost = Attribute::get(
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
>From 2081f59a7c7734f9660e0c18cbd88d7a59c5d433 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 21:28:00 +0800
Subject: [PATCH 3/8] Reduce extra cost by half
---
llvm/lib/Transforms/IPO/Inliner.cpp | 7 ++++++-
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 7 ++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 760d8e21d37e8e..655b2e0a7bd6d0 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,6 +376,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
getStringFnAttrAsInt(
*CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
.value_or(1);
+ int CBInliningExtraCost =
+ getStringFnAttrAsInt(
+ *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+ .value_or(0);
std::optional<int> InliningCost = Advice->inliningCost();
// Setup the data structure used to plumb customization into the
@@ -440,7 +444,8 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
Attribute NewCBExtraCost = Attribute::get(
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
- itostr(*InliningCost));
+ itostr(CBInliningExtraCost +
+ (*InliningCost - CBInliningExtraCost) / 2));
ICB->addFnAttr(NewCBExtraCost);
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index cee31e30e94d46..5e15b0216bc3e5 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,6 +225,10 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
Advice->recordUnattemptedInlining();
continue;
}
+ int CBInliningExtraCost =
+ getStringFnAttrAsInt(
+ *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+ .value_or(0);
std::optional<int> InliningCost = Advice->inliningCost();
// Setup the data structure used to plumb customization into the
@@ -272,7 +276,8 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
Attribute NewCBExtraCost = Attribute::get(
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
- itostr(*InliningCost));
+ itostr(CBInliningExtraCost +
+ (*InliningCost - CBInliningExtraCost) / 2));
ICB->addFnAttr(NewCBExtraCost);
}
}
>From b8db1f5e7898c99167c19d0aaa9722571d975e88 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 22:06:04 +0800
Subject: [PATCH 4/8] Continue to reduce by half
---
llvm/lib/Transforms/IPO/Inliner.cpp | 2 +-
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 655b2e0a7bd6d0..9a473c260fe2a3 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 2));
+ (*InliningCost - CBInliningExtraCost) / 4));
ICB->addFnAttr(NewCBExtraCost);
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 5e15b0216bc3e5..e4be6465374cd9 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 2));
+ (*InliningCost - CBInliningExtraCost) / 4));
ICB->addFnAttr(NewCBExtraCost);
}
}
>From 5900ba558b2ce7a45e3182a9ca3a05e63a3bb306 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 22:39:05 +0800
Subject: [PATCH 5/8] Continue to reduce by half
---
llvm/lib/Transforms/IPO/Inliner.cpp | 2 +-
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 9a473c260fe2a3..ec6f3f907b7fa6 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 4));
+ (*InliningCost - CBInliningExtraCost) / 8));
ICB->addFnAttr(NewCBExtraCost);
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index e4be6465374cd9..46b00c20478ce3 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 4));
+ (*InliningCost - CBInliningExtraCost) / 8));
ICB->addFnAttr(NewCBExtraCost);
}
}
>From 5287a53e9f66828616fedd770f65524d93f2fd46 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Thu, 3 Oct 2024 13:00:27 +0800
Subject: [PATCH 6/8] Continue to reduce by half
---
llvm/lib/Transforms/IPO/Inliner.cpp | 2 +-
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index ec6f3f907b7fa6..059b7dbf81a6bf 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 8));
+ (*InliningCost - CBInliningExtraCost) / 16));
ICB->addFnAttr(NewCBExtraCost);
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 46b00c20478ce3..7605449fba46ae 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
M.getContext(),
InlineConstants::FunctionInlineExtraCostAttributeName,
itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 8));
+ (*InliningCost - CBInliningExtraCost) / 16));
ICB->addFnAttr(NewCBExtraCost);
}
}
>From e6e7dd9a44c568341cf7c93c3fdf92274a918375 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Fri, 4 Oct 2024 14:28:36 +0800
Subject: [PATCH 7/8] Rename Extra to Aditional
---
llvm/include/llvm/Analysis/InlineCost.h | 6 +++---
llvm/lib/Analysis/InlineCost.cpp | 6 +++---
llvm/lib/Transforms/IPO/Inliner.cpp | 17 ++++++++++-------
llvm/lib/Transforms/IPO/ModuleInliner.cpp | 17 ++++++++++-------
4 files changed, 26 insertions(+), 20 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 44b824305b3f7b..1190308dba4193 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -58,9 +58,9 @@ const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536;
const char FunctionInlineCostMultiplierAttributeName[] =
"function-inline-cost-multiplier";
-
-const char FunctionInlineExtraCostAttributeName[] =
- "function-inline-extra-cost";
+/// Cost of call site accumulation added after inlining.
+const char FunctionInlineAdditionalCostAttributeName[] =
+ "function-inline-additional-cost";
const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize";
} // namespace InlineConstants
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index c379d6a08609bf..e45423f2130d8a 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1017,10 +1017,10 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
InlineConstants::FunctionInlineCostMultiplierAttributeName))
Cost *= *AttrCostMult;
- if (std::optional<int> AttrExtraCost = getStringFnAttrAsInt(
+ if (std::optional<int> AttrAdditonalCost = getStringFnAttrAsInt(
CandidateCall,
- InlineConstants::FunctionInlineExtraCostAttributeName))
- Cost += *AttrExtraCost;
+ InlineConstants::FunctionInlineAdditionalCostAttributeName))
+ Cost += *AttrAdditonalCost;
if (std::optional<int> AttrThreshold =
getStringFnAttrAsInt(CandidateCall, "function-inline-threshold"))
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 059b7dbf81a6bf..977bbcd35f73e8 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,9 +376,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
getStringFnAttrAsInt(
*CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
.value_or(1);
- int CBInliningExtraCost =
+ int CBInliningAdditionalCost =
getStringFnAttrAsInt(
- *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+ *CB, InlineConstants::FunctionInlineAdditionalCostAttributeName)
.value_or(0);
std::optional<int> InliningCost = Advice->inliningCost();
@@ -441,12 +441,15 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
itostr(CBCostMult * IntraSCCCostMultiplier));
ICB->addFnAttr(NewCBCostMult);
} else if (InliningCost && *InliningCost > 0) {
- Attribute NewCBExtraCost = Attribute::get(
+ // Similar to hot call site thresholds that can cause Inliner to
+ // inline numerous functions causing compile time issues, a
+ // linear accumulator was created to mitigate the problem.
+ Attribute NewCBAdditionalCost = Attribute::get(
M.getContext(),
- InlineConstants::FunctionInlineExtraCostAttributeName,
- itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 16));
- ICB->addFnAttr(NewCBExtraCost);
+ InlineConstants::FunctionInlineAdditionalCostAttributeName,
+ itostr(CBInliningAdditionalCost +
+ (*InliningCost - CBInliningAdditionalCost) / 16));
+ ICB->addFnAttr(NewCBAdditionalCost);
}
}
}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 7605449fba46ae..c196be9d6dd163 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,9 +225,9 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
Advice->recordUnattemptedInlining();
continue;
}
- int CBInliningExtraCost =
+ int CBInliningAdditionalCost =
getStringFnAttrAsInt(
- *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+ *CB, InlineConstants::FunctionInlineAdditionalCostAttributeName)
.value_or(0);
std::optional<int> InliningCost = Advice->inliningCost();
@@ -273,12 +273,15 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
if (!NewCallee->isDeclaration()) {
Calls->push({ICB, NewHistoryID});
if (InliningCost && *InliningCost > 0) {
- Attribute NewCBExtraCost = Attribute::get(
+ // Similar to hot call site thresholds that can cause Inliner to
+ // inline numerous functions causing compile time issues, a linear
+ // accumulator was created to mitigate the problem.
+ Attribute NewCBAdditionalCost = Attribute::get(
M.getContext(),
- InlineConstants::FunctionInlineExtraCostAttributeName,
- itostr(CBInliningExtraCost +
- (*InliningCost - CBInliningExtraCost) / 16));
- ICB->addFnAttr(NewCBExtraCost);
+ InlineConstants::FunctionInlineAdditionalCostAttributeName,
+ itostr(CBInliningAdditionalCost +
+ (*InliningCost - CBInliningAdditionalCost) / 16));
+ ICB->addFnAttr(NewCBAdditionalCost);
}
}
}
>From f3517f752c3dbcc60767cbbfa959ab0eefcaf0cc Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Fri, 4 Oct 2024 14:45:42 +0800
Subject: [PATCH 8/8] Add a test case
---
.../Inline/inline-hot-callsite-limit.ll | 109 ++++++++++++++++++
1 file changed, 109 insertions(+)
create mode 100644 llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll
diff --git a/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll b/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll
new file mode 100644
index 00000000000000..a1730d76cd547c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; This tests that a hot callsite gets the (higher) inlinehint-threshold even without
+; without inline hints and gets inlined because the cost is less than
+; RUN: opt < %s -passes=inline -inline-threshold=0 -locally-hot-callsite-threshold=30 -S | FileCheck %s
+; RUN: opt < %s -passes=module-inline -inline-threshold=0 -locally-hot-callsite-threshold=30 -S | FileCheck %s
+
+; Due to the hot call site, foo0 inlined foo1, foo2, and foo3,
+; but foo4 is not inlined due to the accumulated cost.
+
+declare void @bar(ptr)
+
+define void @foo0(ptr %p) {
+; CHECK-LABEL: define void @foo0(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[HEADER:.*:]]
+; CHECK-NEXT: [[I_I2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I_I1:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I_I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT: call void @bar(ptr [[I_I]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I1]])
+; CHECK-NEXT: call void @bar(ptr [[I_I1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I2]])
+; CHECK-NEXT: call void @bar(ptr [[I_I2]])
+; CHECK-NEXT: call void @foo4(ptr [[P]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I1]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT: br label %[[LOOP]]
+;
+header:
+ br label %loop
+
+loop:
+ call void @foo1(ptr %p)
+ br label %loop
+}
+
+define void @foo1(ptr %p) {
+; CHECK-LABEL: define void @foo1(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: call void @foo2(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ %i = alloca i32
+ call void @bar(ptr %i)
+ call void @foo2(ptr %p)
+ ret void
+}
+
+define void @foo2(ptr %p) {
+; CHECK-LABEL: define void @foo2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: call void @foo3(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ %i = alloca i32
+ call void @bar(ptr %i)
+ call void @foo3(ptr %p)
+ ret void
+}
+
+define void @foo3(ptr %p) {
+; CHECK-LABEL: define void @foo3(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: call void @foo4(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ %i = alloca i32
+ call void @bar(ptr %i)
+ call void @foo4(ptr %p)
+ ret void
+}
+
+define void @foo4(ptr %p) {
+; CHECK-LABEL: define void @foo4(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: call void @foo5(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ %i = alloca i32
+ call void @bar(ptr %i)
+ call void @foo5(ptr %p)
+ ret void
+}
+
+define void @foo5(ptr %p) {
+; CHECK-LABEL: define void @foo5(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: call void @bar(ptr [[I]])
+; CHECK-NEXT: ret void
+;
+ %i = alloca i32
+ call void @bar(ptr %i)
+ call void @bar(ptr %i)
+ ret void
+}
More information about the llvm-commits
mailing list