[llvm] [Inline] Accumulate the cost of the inlined function to the new call site (PR #111104)

via llvm-commits llvm-commits at lists.llvm.org
Fri Oct 4 00:21:21 PDT 2024


https://github.com/DianQK created https://github.com/llvm/llvm-project/pull/111104

Fixes #111102.

Consider the following IR, the `call void @bar` is a local host call site:

```llvm
@define void foo() {
  ...
loop:
  call void @bar()
  ...
}

define void @bar() {
  ...
  call void @baz()
  ...
}

define void @baz() {
  ...
}
```

If `foo` can inline `baz` after inlined `bar`, I think it can also inline a `bar` that has inlined `baz`. With this in mind, I accumulated the previous cost into the new call site, which is a linearly increasing limit.

I did some experimenting on [DianQK/perf/inline-extra-cost](https://llvm-compile-time-tracker.com/?config=Overview&stat=instructions%3Au&remote=DianQK) and the current thresholds look OK. I'm looking forward to getting some feedback before modifying this threshold to an option.

>From 558f121259136d91ca758fe5acd61038f215acb6 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Tue, 1 Oct 2024 19:46:28 +0800
Subject: [PATCH 1/8] [Inline] Accumulate the cost of the inlined function to
 the new call site

---
 llvm/include/llvm/Analysis/InlineAdvisor.h           | 12 +++++++++---
 llvm/include/llvm/Analysis/InlineCost.h              |  3 +++
 llvm/lib/Analysis/InlineAdvisor.cpp                  |  6 ++++--
 llvm/lib/Analysis/InlineCost.cpp                     |  5 +++++
 llvm/lib/Transforms/IPO/Inliner.cpp                  |  7 +++++++
 llvm/lib/Transforms/IPO/ModuleInliner.cpp            | 11 ++++++++++-
 .../Transforms/Inline/inline-history-noinline.ll     |  2 +-
 7 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 871a6e97861e29..2880553817ebd8 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -74,7 +74,8 @@ class InlineAdvisor;
 class InlineAdvice {
 public:
   InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
-               OptimizationRemarkEmitter &ORE, bool IsInliningRecommended);
+               OptimizationRemarkEmitter &ORE, bool IsInliningRecommended,
+               std::optional<int> InliningCost = std::nullopt);
 
   InlineAdvice(InlineAdvice &&) = delete;
   InlineAdvice(const InlineAdvice &) = delete;
@@ -108,6 +109,7 @@ class InlineAdvice {
 
   /// Get the inlining recommendation.
   bool isInliningRecommended() const { return IsInliningRecommended; }
+  std::optional<int> inliningCost() const { return InliningCost; }
   const DebugLoc &getOriginalCallSiteDebugLoc() const { return DLoc; }
   const BasicBlock *getOriginalCallSiteBasicBlock() const { return Block; }
 
@@ -129,6 +131,7 @@ class InlineAdvice {
   const BasicBlock *const Block;
   OptimizationRemarkEmitter &ORE;
   const bool IsInliningRecommended;
+  const std::optional<int> InliningCost;
 
 private:
   void markRecorded() {
@@ -145,8 +148,11 @@ class DefaultInlineAdvice : public InlineAdvice {
   DefaultInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
                       std::optional<InlineCost> OIC,
                       OptimizationRemarkEmitter &ORE, bool EmitRemarks = true)
-      : InlineAdvice(Advisor, CB, ORE, OIC.has_value()), OriginalCB(&CB),
-        OIC(OIC), EmitRemarks(EmitRemarks) {}
+      : InlineAdvice(Advisor, CB, ORE, OIC.has_value(),
+                     OIC && OIC->isVariable()
+                         ? std::optional<int>(OIC->getCost())
+                         : std::nullopt),
+        OriginalCB(&CB), OIC(OIC), EmitRemarks(EmitRemarks) {}
 
 private:
   void recordUnsuccessfulInliningImpl(const InlineResult &Result) override;
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index c5978ce54fc18b..44b824305b3f7b 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -59,6 +59,9 @@ const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536;
 const char FunctionInlineCostMultiplierAttributeName[] =
     "function-inline-cost-multiplier";
 
+const char FunctionInlineExtraCostAttributeName[] =
+    "function-inline-extra-cost";
+
 const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize";
 } // namespace InlineConstants
 
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index c6907cb128bb47..9706dd212d7bc5 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -175,10 +175,12 @@ DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
 
 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
                            OptimizationRemarkEmitter &ORE,
-                           bool IsInliningRecommended)
+                           bool IsInliningRecommended,
+                           std::optional<int> InliningCost)
     : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
       DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
-      IsInliningRecommended(IsInliningRecommended) {}
+      IsInliningRecommended(IsInliningRecommended), InliningCost(InliningCost) {
+}
 
 void InlineAdvice::recordInlineStatsIfNeeded() {
   if (Advisor->ImportedFunctionsStats)
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index d2c329ba748e58..c379d6a08609bf 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1017,6 +1017,11 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
             InlineConstants::FunctionInlineCostMultiplierAttributeName))
       Cost *= *AttrCostMult;
 
+    if (std::optional<int> AttrExtraCost = getStringFnAttrAsInt(
+            CandidateCall,
+            InlineConstants::FunctionInlineExtraCostAttributeName))
+      Cost += *AttrExtraCost;
+
     if (std::optional<int> AttrThreshold =
             getStringFnAttrAsInt(CandidateCall, "function-inline-threshold"))
       Threshold = *AttrThreshold;
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 23ee23eb047f58..746cc34b70b16d 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,6 +376,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
           getStringFnAttrAsInt(
               *CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
               .value_or(1);
+      std::optional<int> InliningCost = Advice->inliningCost();
 
       // Setup the data structure used to plumb customization into the
       // `InlineFunction` routine.
@@ -435,6 +436,12 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     InlineConstants::FunctionInlineCostMultiplierAttributeName,
                     itostr(CBCostMult * IntraSCCCostMultiplier));
                 ICB->addFnAttr(NewCBCostMult);
+              } else if (InliningCost) {
+                Attribute NewCBExtraCost = Attribute::get(
+                    M.getContext(),
+                    InlineConstants::FunctionInlineExtraCostAttributeName,
+                    itostr(*InliningCost));
+                ICB->addFnAttr(NewCBExtraCost);
               }
             }
           }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index dbc733826944b9..71937ff1172c07 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,6 +225,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
       Advice->recordUnattemptedInlining();
       continue;
     }
+    std::optional<int> InliningCost = Advice->inliningCost();
 
     // Setup the data structure used to plumb customization into the
     // `InlineFunction` routine.
@@ -265,8 +266,16 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
               NewCallee = ICB->getCalledFunction();
         }
         if (NewCallee)
-          if (!NewCallee->isDeclaration())
+          if (!NewCallee->isDeclaration()) {
             Calls->push({ICB, NewHistoryID});
+            if (InliningCost) {
+              Attribute NewCBExtraCost = Attribute::get(
+                  M.getContext(),
+                  InlineConstants::FunctionInlineExtraCostAttributeName,
+                  itostr(*InliningCost));
+              ICB->addFnAttr(NewCBExtraCost);
+            }
+          }
       }
     }
 
diff --git a/llvm/test/Transforms/Inline/inline-history-noinline.ll b/llvm/test/Transforms/Inline/inline-history-noinline.ll
index 742bd25ecd9bb9..fbe633fc3c797b 100644
--- a/llvm/test/Transforms/Inline/inline-history-noinline.ll
+++ b/llvm/test/Transforms/Inline/inline-history-noinline.ll
@@ -29,4 +29,4 @@ define internal void @a() {
   ret void
 }
 
-; CHECK: [[NOINLINE]] = { noinline }
+; CHECK: [[NOINLINE]] = { noinline {{.*}}}

>From d133af8bb0adc2f8bf4a44b20ba6ad594988efd2 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 19:21:29 +0800
Subject: [PATCH 2/8] Only apply when cost is positive

---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 2 +-
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 746cc34b70b16d..760d8e21d37e8e 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -436,7 +436,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     InlineConstants::FunctionInlineCostMultiplierAttributeName,
                     itostr(CBCostMult * IntraSCCCostMultiplier));
                 ICB->addFnAttr(NewCBCostMult);
-              } else if (InliningCost) {
+              } else if (InliningCost && *InliningCost > 0) {
                 Attribute NewCBExtraCost = Attribute::get(
                     M.getContext(),
                     InlineConstants::FunctionInlineExtraCostAttributeName,
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 71937ff1172c07..cee31e30e94d46 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -268,7 +268,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
         if (NewCallee)
           if (!NewCallee->isDeclaration()) {
             Calls->push({ICB, NewHistoryID});
-            if (InliningCost) {
+            if (InliningCost && *InliningCost > 0) {
               Attribute NewCBExtraCost = Attribute::get(
                   M.getContext(),
                   InlineConstants::FunctionInlineExtraCostAttributeName,

>From 2081f59a7c7734f9660e0c18cbd88d7a59c5d433 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 21:28:00 +0800
Subject: [PATCH 3/8] Reduce extra cost by half

---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 7 ++++++-
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 7 ++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 760d8e21d37e8e..655b2e0a7bd6d0 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,6 +376,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
           getStringFnAttrAsInt(
               *CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
               .value_or(1);
+      int CBInliningExtraCost =
+          getStringFnAttrAsInt(
+              *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+              .value_or(0);
       std::optional<int> InliningCost = Advice->inliningCost();
 
       // Setup the data structure used to plumb customization into the
@@ -440,7 +444,8 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                 Attribute NewCBExtraCost = Attribute::get(
                     M.getContext(),
                     InlineConstants::FunctionInlineExtraCostAttributeName,
-                    itostr(*InliningCost));
+                    itostr(CBInliningExtraCost +
+                           (*InliningCost - CBInliningExtraCost) / 2));
                 ICB->addFnAttr(NewCBExtraCost);
               }
             }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index cee31e30e94d46..5e15b0216bc3e5 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,6 +225,10 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
       Advice->recordUnattemptedInlining();
       continue;
     }
+    int CBInliningExtraCost =
+        getStringFnAttrAsInt(
+            *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+            .value_or(0);
     std::optional<int> InliningCost = Advice->inliningCost();
 
     // Setup the data structure used to plumb customization into the
@@ -272,7 +276,8 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
               Attribute NewCBExtraCost = Attribute::get(
                   M.getContext(),
                   InlineConstants::FunctionInlineExtraCostAttributeName,
-                  itostr(*InliningCost));
+                  itostr(CBInliningExtraCost +
+                         (*InliningCost - CBInliningExtraCost) / 2));
               ICB->addFnAttr(NewCBExtraCost);
             }
           }

>From b8db1f5e7898c99167c19d0aaa9722571d975e88 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 22:06:04 +0800
Subject: [PATCH 4/8] Continue to reduce by half

---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 2 +-
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 655b2e0a7bd6d0..9a473c260fe2a3 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     M.getContext(),
                     InlineConstants::FunctionInlineExtraCostAttributeName,
                     itostr(CBInliningExtraCost +
-                           (*InliningCost - CBInliningExtraCost) / 2));
+                           (*InliningCost - CBInliningExtraCost) / 4));
                 ICB->addFnAttr(NewCBExtraCost);
               }
             }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 5e15b0216bc3e5..e4be6465374cd9 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
                   M.getContext(),
                   InlineConstants::FunctionInlineExtraCostAttributeName,
                   itostr(CBInliningExtraCost +
-                         (*InliningCost - CBInliningExtraCost) / 2));
+                         (*InliningCost - CBInliningExtraCost) / 4));
               ICB->addFnAttr(NewCBExtraCost);
             }
           }

>From 5900ba558b2ce7a45e3182a9ca3a05e63a3bb306 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Wed, 2 Oct 2024 22:39:05 +0800
Subject: [PATCH 5/8] Continue to reduce by half

---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 2 +-
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 9a473c260fe2a3..ec6f3f907b7fa6 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     M.getContext(),
                     InlineConstants::FunctionInlineExtraCostAttributeName,
                     itostr(CBInliningExtraCost +
-                           (*InliningCost - CBInliningExtraCost) / 4));
+                           (*InliningCost - CBInliningExtraCost) / 8));
                 ICB->addFnAttr(NewCBExtraCost);
               }
             }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index e4be6465374cd9..46b00c20478ce3 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
                   M.getContext(),
                   InlineConstants::FunctionInlineExtraCostAttributeName,
                   itostr(CBInliningExtraCost +
-                         (*InliningCost - CBInliningExtraCost) / 4));
+                         (*InliningCost - CBInliningExtraCost) / 8));
               ICB->addFnAttr(NewCBExtraCost);
             }
           }

>From 5287a53e9f66828616fedd770f65524d93f2fd46 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Thu, 3 Oct 2024 13:00:27 +0800
Subject: [PATCH 6/8] Continue to reduce by half

---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 2 +-
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index ec6f3f907b7fa6..059b7dbf81a6bf 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -445,7 +445,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     M.getContext(),
                     InlineConstants::FunctionInlineExtraCostAttributeName,
                     itostr(CBInliningExtraCost +
-                           (*InliningCost - CBInliningExtraCost) / 8));
+                           (*InliningCost - CBInliningExtraCost) / 16));
                 ICB->addFnAttr(NewCBExtraCost);
               }
             }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 46b00c20478ce3..7605449fba46ae 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -277,7 +277,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
                   M.getContext(),
                   InlineConstants::FunctionInlineExtraCostAttributeName,
                   itostr(CBInliningExtraCost +
-                         (*InliningCost - CBInliningExtraCost) / 8));
+                         (*InliningCost - CBInliningExtraCost) / 16));
               ICB->addFnAttr(NewCBExtraCost);
             }
           }

>From e6e7dd9a44c568341cf7c93c3fdf92274a918375 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Fri, 4 Oct 2024 14:28:36 +0800
Subject: [PATCH 7/8] Rename Extra to Aditional

---
 llvm/include/llvm/Analysis/InlineCost.h   |  6 +++---
 llvm/lib/Analysis/InlineCost.cpp          |  6 +++---
 llvm/lib/Transforms/IPO/Inliner.cpp       | 17 ++++++++++-------
 llvm/lib/Transforms/IPO/ModuleInliner.cpp | 17 ++++++++++-------
 4 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 44b824305b3f7b..1190308dba4193 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -58,9 +58,9 @@ const uint64_t MaxSimplifiedDynamicAllocaToInline = 65536;
 
 const char FunctionInlineCostMultiplierAttributeName[] =
     "function-inline-cost-multiplier";
-
-const char FunctionInlineExtraCostAttributeName[] =
-    "function-inline-extra-cost";
+/// Cost of call site accumulation added after inlining.
+const char FunctionInlineAdditionalCostAttributeName[] =
+    "function-inline-additional-cost";
 
 const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize";
 } // namespace InlineConstants
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index c379d6a08609bf..e45423f2130d8a 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1017,10 +1017,10 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
             InlineConstants::FunctionInlineCostMultiplierAttributeName))
       Cost *= *AttrCostMult;
 
-    if (std::optional<int> AttrExtraCost = getStringFnAttrAsInt(
+    if (std::optional<int> AttrAdditonalCost = getStringFnAttrAsInt(
             CandidateCall,
-            InlineConstants::FunctionInlineExtraCostAttributeName))
-      Cost += *AttrExtraCost;
+            InlineConstants::FunctionInlineAdditionalCostAttributeName))
+      Cost += *AttrAdditonalCost;
 
     if (std::optional<int> AttrThreshold =
             getStringFnAttrAsInt(CandidateCall, "function-inline-threshold"))
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 059b7dbf81a6bf..977bbcd35f73e8 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -376,9 +376,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
           getStringFnAttrAsInt(
               *CB, InlineConstants::FunctionInlineCostMultiplierAttributeName)
               .value_or(1);
-      int CBInliningExtraCost =
+      int CBInliningAdditionalCost =
           getStringFnAttrAsInt(
-              *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+              *CB, InlineConstants::FunctionInlineAdditionalCostAttributeName)
               .value_or(0);
       std::optional<int> InliningCost = Advice->inliningCost();
 
@@ -441,12 +441,15 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                     itostr(CBCostMult * IntraSCCCostMultiplier));
                 ICB->addFnAttr(NewCBCostMult);
               } else if (InliningCost && *InliningCost > 0) {
-                Attribute NewCBExtraCost = Attribute::get(
+                // Similar to hot call site thresholds that can cause Inliner to
+                // inline numerous functions causing compile time issues, a
+                // linear accumulator was created to mitigate the problem.
+                Attribute NewCBAdditionalCost = Attribute::get(
                     M.getContext(),
-                    InlineConstants::FunctionInlineExtraCostAttributeName,
-                    itostr(CBInliningExtraCost +
-                           (*InliningCost - CBInliningExtraCost) / 16));
-                ICB->addFnAttr(NewCBExtraCost);
+                    InlineConstants::FunctionInlineAdditionalCostAttributeName,
+                    itostr(CBInliningAdditionalCost +
+                           (*InliningCost - CBInliningAdditionalCost) / 16));
+                ICB->addFnAttr(NewCBAdditionalCost);
               }
             }
           }
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 7605449fba46ae..c196be9d6dd163 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -225,9 +225,9 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
       Advice->recordUnattemptedInlining();
       continue;
     }
-    int CBInliningExtraCost =
+    int CBInliningAdditionalCost =
         getStringFnAttrAsInt(
-            *CB, InlineConstants::FunctionInlineExtraCostAttributeName)
+            *CB, InlineConstants::FunctionInlineAdditionalCostAttributeName)
             .value_or(0);
     std::optional<int> InliningCost = Advice->inliningCost();
 
@@ -273,12 +273,15 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
           if (!NewCallee->isDeclaration()) {
             Calls->push({ICB, NewHistoryID});
             if (InliningCost && *InliningCost > 0) {
-              Attribute NewCBExtraCost = Attribute::get(
+              // Similar to hot call site thresholds that can cause Inliner to
+              // inline numerous functions causing compile time issues, a linear
+              // accumulator was created to mitigate the problem.
+              Attribute NewCBAdditionalCost = Attribute::get(
                   M.getContext(),
-                  InlineConstants::FunctionInlineExtraCostAttributeName,
-                  itostr(CBInliningExtraCost +
-                         (*InliningCost - CBInliningExtraCost) / 16));
-              ICB->addFnAttr(NewCBExtraCost);
+                  InlineConstants::FunctionInlineAdditionalCostAttributeName,
+                  itostr(CBInliningAdditionalCost +
+                         (*InliningCost - CBInliningAdditionalCost) / 16));
+              ICB->addFnAttr(NewCBAdditionalCost);
             }
           }
       }

>From f3517f752c3dbcc60767cbbfa959ab0eefcaf0cc Mon Sep 17 00:00:00 2001
From: DianQK <dianqk at dianqk.net>
Date: Fri, 4 Oct 2024 14:45:42 +0800
Subject: [PATCH 8/8] Add a test case

---
 .../Inline/inline-hot-callsite-limit.ll       | 109 ++++++++++++++++++
 1 file changed, 109 insertions(+)
 create mode 100644 llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll

diff --git a/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll b/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll
new file mode 100644
index 00000000000000..a1730d76cd547c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-hot-callsite-limit.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; This tests that a hot callsite gets the (higher) inlinehint-threshold even without
+; without inline hints and gets inlined because the cost is less than
+; RUN: opt < %s -passes=inline -inline-threshold=0 -locally-hot-callsite-threshold=30 -S | FileCheck %s
+; RUN: opt < %s -passes=module-inline -inline-threshold=0 -locally-hot-callsite-threshold=30 -S | FileCheck %s
+
+; Due to the hot call site, foo0 inlined foo1, foo2, and foo3,
+; but foo4 is not inlined due to the accumulated cost.
+
+declare void @bar(ptr)
+
+define void @foo0(ptr %p) {
+; CHECK-LABEL: define void @foo0(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[HEADER:.*:]]
+; CHECK-NEXT:    [[I_I2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[I_I1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[I_I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT:    call void @bar(ptr [[I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I1]])
+; CHECK-NEXT:    call void @bar(ptr [[I_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I2]])
+; CHECK-NEXT:    call void @bar(ptr [[I_I2]])
+; CHECK-NEXT:    call void @foo4(ptr [[P]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT:    br label %[[LOOP]]
+;
+header:
+  br label %loop
+
+loop:
+  call void @foo1(ptr %p)
+  br label %loop
+}
+
+define void @foo1(ptr %p) {
+; CHECK-LABEL: define void @foo1(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    call void @foo2(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %i = alloca i32
+  call void @bar(ptr %i)
+  call void @foo2(ptr %p)
+  ret void
+}
+
+define void @foo2(ptr %p) {
+; CHECK-LABEL: define void @foo2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    call void @foo3(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %i = alloca i32
+  call void @bar(ptr %i)
+  call void @foo3(ptr %p)
+  ret void
+}
+
+define void @foo3(ptr %p) {
+; CHECK-LABEL: define void @foo3(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    call void @foo4(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %i = alloca i32
+  call void @bar(ptr %i)
+  call void @foo4(ptr %p)
+  ret void
+}
+
+define void @foo4(ptr %p) {
+; CHECK-LABEL: define void @foo4(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    call void @foo5(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %i = alloca i32
+  call void @bar(ptr %i)
+  call void @foo5(ptr %p)
+  ret void
+}
+
+define void @foo5(ptr %p) {
+; CHECK-LABEL: define void @foo5(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    call void @bar(ptr [[I]])
+; CHECK-NEXT:    ret void
+;
+  %i = alloca i32
+  call void @bar(ptr %i)
+  call void @bar(ptr %i)
+  ret void
+}



More information about the llvm-commits mailing list