[llvm] [ModuleInliner] Skip dead basic blocks during inlining (PR #195799)

[Kazu Hirata via llvm-commits]

https://github.com/kazutakahirata created https://github.com/llvm/llvm-project/pull/195799

In the module inliner, inlining a function can cause certain basic
blocks to become unreachable from the entry block.  These dead blocks
pollute the worklist queue with dead call sites, causing the pass
to inline callees into dead basic blocks.

This patch tracks unreachable blocks in a set, allowing the pass to
lazily skip call sites in dead blocks.  Note that the downstream
SimplifyCFG pass cleans up these dead basic blocks.  We've decided
against removing these dead basic blocks in the inlining loop; doing so
would require us to use WeakTrackingVH and frequently clear analyses,
slowing down the compiler.


>From 0690111fceab9b2cfb2960330c0acfecf989602f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Mon, 4 May 2026 21:27:08 -0700
Subject: [PATCH] [ModuleInliner] Skip dead basic blocks during inlining

In the module inliner, inlining a function can cause certain basic
blocks to become unreachable from the entry block.  These dead blocks
pollute the worklist queue with dead call sites, causing the pass
to inline callees into dead basic blocks.

This patch tracks unreachable blocks in a set, allowing the pass to
lazily skip call sites in dead blocks.  Note that the downstream
SimplifyCFG pass cleans up these dead basic blocks.  We've decided
against removing these dead basic blocks in the inlining loop; doing so
would require us to use WeakTrackingVH and frequently clear analyses,
slowing down the compiler.
---
 llvm/lib/Transforms/IPO/ModuleInliner.cpp     | 24 ++++++++++
 .../Inline/module-inliner-skip-dead-blocks.ll | 47 +++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 llvm/test/Transforms/Inline/module-inliner-skip-dead-blocks.ll

diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index bd244a9f4fd27..58812b96b2f68 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -93,6 +93,22 @@ static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) {
          TLI.isKnownVectorFunctionInLibrary(F.getName());
 }
 
+/// Collects basic blocks unreachable from the entry block into DeadBlocks.
+static void
+recordDeadBasicBlocks(Function &F,
+                      SmallPtrSetImpl<const BasicBlock *> &DeadBlocks) {
+  SmallSetVector<const BasicBlock *, 8> Reachable;
+  Reachable.insert(&F.getEntryBlock());
+
+  for (unsigned Idx = 0; Idx < Reachable.size(); ++Idx)
+    for (const BasicBlock *Succ : successors(Reachable[Idx]))
+      Reachable.insert(Succ);
+
+  for (BasicBlock &BB : F)
+    if (!Reachable.contains(&BB))
+      DeadBlocks.insert(&BB);
+}
+
 PreservedAnalyses ModuleInlinerPass::run(Module &M,
                                          ModuleAnalysisManager &MAM) {
   LLVM_DEBUG(dbgs() << "---- Module Inliner is Running ---- \n");
@@ -177,9 +193,15 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
   // defer deleting these to make it easier to handle the call graph updates.
   SmallVector<Function *, 4> DeadFunctions;
 
+  // Track the dead basic blocks to prune once finished with inlining calls. We
+  // defer pruning these to avoid dangling pointers in the worklist queue.
+  SmallPtrSet<const BasicBlock *, 8> DeadBlocks;
+
   // Loop forward over all of the calls.
   while (!Calls->empty()) {
     CallBase *CB = Calls->pop();
+    if (DeadBlocks.count(CB->getParent()))
+      continue;
     Function &F = *CB->getCaller();
     Function &Callee = *CB->getCalledFunction();
 
@@ -272,6 +294,8 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
     }
     if (!CalleeWasDeleted)
       Advice->recordInlining();
+
+    recordDeadBasicBlocks(F, DeadBlocks);
   }
 
   // Now that we've finished inlining all of the calls across this module,
diff --git a/llvm/test/Transforms/Inline/module-inliner-skip-dead-blocks.ll b/llvm/test/Transforms/Inline/module-inliner-skip-dead-blocks.ll
new file mode 100644
index 0000000000000..ff890de8e5a41
--- /dev/null
+++ b/llvm/test/Transforms/Inline/module-inliner-skip-dead-blocks.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='module-inline' -inline-priority-mode=cost-benefit -S < %s | FileCheck %s
+
+; Verify that the inliner does nothing for the call to @terminate after
+; inlining @callee renders its containing basic block dead.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+define void @test_skip_dead_blocks() personality ptr null {
+; CHECK-LABEL: define void @test_skip_dead_blocks() personality ptr null {
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BB2:.*:]]
+; CHECK-NEXT:    [[TMP3:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:            catch ptr null
+; CHECK-NEXT:    call void @terminate()
+; CHECK-NEXT:    unreachable
+;
+  %1 = invoke ptr @callee(ptr null, ptr null, ptr null)
+  to label %2 unwind label %3
+
+2:                                                ; preds = %0
+  ret void
+
+3:                                                ; preds = %0
+  %4 = landingpad { ptr, i32 }
+  catch ptr null
+  call void @terminate()
+  unreachable
+}
+
+define ptr @callee(ptr %0, ptr %1, ptr %2) {
+; CHECK-LABEL: define ptr @callee(
+; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], ptr [[TMP2:%.*]]) {
+; CHECK-NEXT:    ret ptr [[TMP0]]
+;
+  ret ptr %0
+}
+
+define void @terminate() {
+; CHECK-LABEL: define void @terminate() {
+; CHECK-NEXT:    unreachable
+;
+  unreachable
+}