[llvm] [BOLT][BTI] Skip inlining BBs with indirect tailcalls (PR #168403)

[Gergely Bálint via llvm-commits]

https://github.com/bgergely0 updated https://github.com/llvm/llvm-project/pull/168403

>From 064f26025ef4d9f33491e6d8260c9e320eed7b17 Mon Sep 17 00:00:00 2001
From: Gergely Balint <gergely.balint at arm.com>
Date: Wed, 3 Sep 2025 12:46:00 +0000
Subject: [PATCH 1/2] [BOLT][BTI] Skip inlining BBs with indirect tailcalls

In the Inliner pass, we need to convert tailcalls to normal calls
in the BB we want to inline.
These tailcalls can be indirect: in this case we would need to update the BTI
on their TargetBB to keep correctness.

As we don't know the targets of indirect tailcalls, we should skip
inlining such blocks.
---
 bolt/lib/Passes/Inliner.cpp    | 23 ++++++++++++++++++++
 bolt/test/AArch64/inline-bti.s | 38 ++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 bolt/test/AArch64/inline-bti.s

diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp
index 5a7d02a34b4d8..5f4001056bb66 100644
--- a/bolt/lib/Passes/Inliner.cpp
+++ b/bolt/lib/Passes/Inliner.cpp
@@ -491,6 +491,29 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
         }
       }
 
+      // AArch64 BTI:
+      // If the callee has an indirect tailcall (BR), we would transform it to
+      // an indirect call (BLR) in InlineCall. Because of this, we would have to
+      // update the BTI at the target of the tailcall. However, these targets
+      // are not known. Instead, we skip inlining blocks with indirect
+      // tailcalls.
+      auto HasIndirectTailCall = [&](const BinaryFunction &BF) -> bool {
+        for (const auto &BB : BF) {
+          for (const auto &II : BB) {
+            if (BC.MIB->isIndirectBranch(II) && BC.MIB->isTailCall(II)) {
+              return true;
+            }
+          }
+        }
+        return false;
+      };
+
+      if (BC.isAArch64() && BC.usesBTI() &&
+          HasIndirectTailCall(*TargetFunction)) {
+        ++InstIt;
+        continue;
+      }
+
       LLVM_DEBUG(dbgs() << "BOLT-DEBUG: inlining call to " << *TargetFunction
                         << " in " << Function << " : " << BB->getName()
                         << ". Count: " << BB->getKnownExecutionCount()
diff --git a/bolt/test/AArch64/inline-bti.s b/bolt/test/AArch64/inline-bti.s
new file mode 100644
index 0000000000000..62f6ea6f4b63a
--- /dev/null
+++ b/bolt/test/AArch64/inline-bti.s
@@ -0,0 +1,38 @@
+## This test checks that for AArch64 binaries with BTI, we do not inline blocks with indirect tailcalls.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags -O0 %t.o -o %t.exe -Wl,-q -Wl,-z,force-bti
+# RUN: llvm-bolt --inline-all %t.exe -o %t.bolt  | FileCheck %s
+
+# For BTI, we should not inline foo.
+# CHECK-NOT: BOLT-INFO: inlined {{[0-9]+}} calls at {{[0-9]+}} call sites in {{[0-9]+}} iteration(s). Change in binary size: {{[0-9]+}} bytes.
+
+	.text
+	.globl	_Z3fooP1A
+	.type	_Z3fooP1A, at function
+_Z3fooP1A:
+	ldr	x8, [x0]
+	ldr	w0, [x8]
+	br x30
+	.size	_Z3fooP1A, .-_Z3fooP1A
+
+	.globl	_Z3barP1A
+	.type	_Z3barP1A, at function
+_Z3barP1A:
+	stp	x29, x30, [sp, #-16]!
+	mov	x29, sp
+	bl	_Z3fooP1A
+	mul	w0, w0, w0
+	ldp	x29, x30, [sp], #16
+	ret
+	.size	_Z3barP1A, .-_Z3barP1A
+
+	.globl	main
+	.p2align	2
+	.type	main, at function
+main:
+	mov	w0, wzr
+	ret
+	.size	main, .-main

>From dec2eeffe8fb36434a4f3b2d295e73bca758306d Mon Sep 17 00:00:00 2001
From: Gergely Balint <gergely.balint at arm.com>
Date: Wed, 26 Nov 2025 13:50:19 +0000
Subject: [PATCH 2/2] [BOLT] Add dbg message about skipped inline

---
 bolt/lib/Passes/Inliner.cpp    | 3 +++
 bolt/test/AArch64/inline-bti.s | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp
index 5f4001056bb66..0740fcef9102b 100644
--- a/bolt/lib/Passes/Inliner.cpp
+++ b/bolt/lib/Passes/Inliner.cpp
@@ -511,6 +511,9 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
       if (BC.isAArch64() && BC.usesBTI() &&
           HasIndirectTailCall(*TargetFunction)) {
         ++InstIt;
+        LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Skipping inlining block with tailcall"
+                          << " in " << Function << " : " << BB->getName()
+                          << " to keep BTIs consistent.\n");
         continue;
       }
 
diff --git a/bolt/test/AArch64/inline-bti.s b/bolt/test/AArch64/inline-bti.s
index 62f6ea6f4b63a..c0b9ebe632b6c 100644
--- a/bolt/test/AArch64/inline-bti.s
+++ b/bolt/test/AArch64/inline-bti.s
@@ -4,9 +4,10 @@
 
 # RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags -O0 %t.o -o %t.exe -Wl,-q -Wl,-z,force-bti
-# RUN: llvm-bolt --inline-all %t.exe -o %t.bolt  | FileCheck %s
+# RUN: llvm-bolt --inline-all %t.exe -o %t.bolt --debug 2>&1 | FileCheck %s
 
 # For BTI, we should not inline foo.
+# CHECK: BOLT-DEBUG: Skipping inlining block with tailcall in _Z3barP1A : .LBB01 to keep BTIs consistent.
 # CHECK-NOT: BOLT-INFO: inlined {{[0-9]+}} calls at {{[0-9]+}} call sites in {{[0-9]+}} iteration(s). Change in binary size: {{[0-9]+}} bytes.
 
 	.text