[llvm] Refund inline cost for blocks terminating in unreachable (PR #213983)

[via llvm-commits]

https://github.com/stephenduong1004 created https://github.com/llvm/llvm-project/pull/213983

This patch addresses the `FIXME` in `CallAnalyzer::visitUnreachableInst` by refunding the accumulated cost of a basic block if it terminates in an `unreachable` instruction. 

Currently, `unreachable` instructions themselves have zero cost. However, the instructions leading up to them are still added to the inline cost budget. Because `analyzeBlock` bails out early if `shouldStop()` evaluates to true, functions with multiple dead-end panic branches artificially inflate their cost and fail to inline.

This is particularly helpful for Rust when compiled with `panic=abort`. While `panic=unwind` generates `invoke` + `landingpad` blocks, `panic=abort` generates a standard `call` + `unreachable` sequence. 


>From 82802d3f5e65800c7505138d833ad1f56219e40e Mon Sep 17 00:00:00 2001
From: Stephen <stephenduong at google.com>
Date: Tue, 4 Aug 2026 11:24:19 -0400
Subject: [PATCH] refund unreachable inline cost

---
 llvm/lib/Analysis/InlineCost.cpp | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 6cd84376d8a98..bad818a4a8fc2 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -865,6 +865,12 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
     }
 
     auto *TI = BB->getTerminator();
+
+    // Refund the accumulated cost of dead-end panic paths.
+    if (isa<UnreachableInst>(TI)) {
+      addCost(CostAtBBStart - Cost);
+    }
+
     // If we had any successors at this point, than post-inlining is likely to
     // have them as well. Note that we assume any basic blocks which existed
     // due to branches or switches which folded above will also fold after
@@ -2706,9 +2712,6 @@ bool CallAnalyzer::visitCatchReturnInst(CatchReturnInst &CRI) {
 }
 
 bool CallAnalyzer::visitUnreachableInst(UnreachableInst &I) {
-  // FIXME: It might be reasonably to discount the cost of instructions leading
-  // to unreachable as they have the lowest possible impact on both runtime and
-  // code size.
   return true; // No actual code is needed for unreachable.
 }
 
@@ -2737,6 +2740,8 @@ bool CallAnalyzer::visitInstruction(Instruction &I) {
 InlineResult
 CallAnalyzer::analyzeBlock(BasicBlock *BB,
                            const SmallPtrSetImpl<const Value *> &EphValues) {
+  bool IsUnreachablePath = isa<UnreachableInst>(BB->getTerminator());
+
   for (Instruction &I : *BB) {
     // FIXME: Currently, the number of instructions in a function regardless of
     // our ability to simplify them during inline to constants or dead code,
@@ -2753,9 +2758,13 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB,
     if (EphValues.count(&I))
       continue;
 
-    ++NumInstructions;
-    if (isa<ExtractElementInst>(I) || I.getType()->isVectorTy())
-      ++NumVectorInstructions;
+    // Do not increment raw instruction counts for unreachable
+    // paths to prevent penalizing the vector bonus heuristic.
+    if (!IsUnreachablePath) {
+      ++NumInstructions;
+      if (isa<ExtractElementInst>(I) || I.getType()->isVectorTy())
+        ++NumVectorInstructions;
+    }
 
     // If the instruction simplified to a constant, there is no cost to this
     // instruction. Visit the instructions using our InstVisitor to account for
@@ -2814,7 +2823,7 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB,
       return IR;
     }
 
-    if (shouldStop())
+    if (!IsUnreachablePath && shouldStop())
       return InlineResult::failure(
           "Call site analysis is not favorable to inlining.");
   }