[llvm] [BOLT] Preserve interprocedural fall-throughs (PR #219430)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 08:14:38 PDT 2026
https://github.com/WangJee updated https://github.com/llvm/llvm-project/pull/219430
>From 5321724a2ed0fbdbb44fad79bea0960fbd470844 Mon Sep 17 00:00:00 2001
From: "wangjue.wangjue" <wangjue.wangjue at alibaba-inc.com>
Date: Fri, 28 Aug 2026 18:49:39 +0900
Subject: [PATCH] [BOLT] Materialize interprocedural fall-throughs
BOLT CFGs do not represent fall-through edges across BinaryFunction boundaries. Function reordering can therefore separate an unterminated function from its successor and change program behavior.
Recognize only simple functions whose reachable final block ends at the declared function size, followed by valid padding and a rewritable function in the same section. Materialize an explicit tail call so both functions may be moved independently. Add X86 and RISC-V regression coverage, including a terminator negative case.
---
bolt/lib/Rewrite/RewriteInstance.cpp | 67 ++++++++++++++
bolt/test/RISCV/interprocedural-fallthrough.s | 86 ++++++++++++++++++
bolt/test/X86/interprocedural-fallthrough.s | 88 +++++++++++++++++++
3 files changed, 241 insertions(+)
create mode 100644 bolt/test/RISCV/interprocedural-fallthrough.s
create mode 100644 bolt/test/X86/interprocedural-fallthrough.s
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 4486efe926d01..e143b1ae22ff3 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -38,6 +38,8 @@
#include "bolt/Utils/Utils.h"
#include "llvm/ADT/AddressRanges.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
@@ -4098,12 +4100,77 @@ void RewriteInstance::buildFunctionsCFG() {
BC->postProcessSymbolTable();
}
+namespace {
+
+bool isReachableFromPrimaryEntry(const BinaryFunction &Function,
+ const BinaryBasicBlock *TargetBB) {
+ SmallPtrSet<const BinaryBasicBlock *, 16> Visited;
+ SmallVector<const BinaryBasicBlock *, 16> Worklist;
+ Worklist.push_back(&Function.front());
+
+ while (!Worklist.empty()) {
+ const BinaryBasicBlock *BB = Worklist.pop_back_val();
+ if (!Visited.insert(BB).second)
+ continue;
+ if (BB == TargetBB)
+ return true;
+ llvm::append_range(Worklist, BB->successors());
+ }
+
+ return false;
+}
+
+BinaryFunction *getInterproceduralFallThroughTarget(BinaryContext &BC,
+ BinaryFunction &Function) {
+ if (!Function.hasCFG() || !Function.isSimple() || Function.isFragment() ||
+ Function.isIgnored() || Function.getLayout().block_empty())
+ return nullptr;
+
+ const BinaryBasicBlock *LastBB = Function.getLayout().block_back();
+ const MCInst *LastInst = LastBB->getLastNonPseudoInstr();
+ if (!LastInst || LastBB->succ_size() != 0 ||
+ BC.MIB->isTerminator(*LastInst) || BC.MIB->isCall(*LastInst) ||
+ LastBB->getEndOffset() != Function.getSize() ||
+ !BC.hasValidCodePadding(Function) ||
+ !isReachableFromPrimaryEntry(Function, LastBB))
+ return nullptr;
+
+ const uint64_t TargetAddress = Function.getAddress() + Function.getMaxSize();
+ BinaryFunction *Target = BC.getBinaryFunctionAtAddress(TargetAddress);
+ if (!Target || Target == &Function || Target->isPseudo() ||
+ Target->isFragment() || Target->isIgnored() ||
+ Target->getOriginSection() != Function.getOriginSection())
+ return nullptr;
+
+ return Target;
+}
+
+} // namespace
+
void RewriteInstance::postProcessFunctions() {
// We mark fragments as non-simple here, not during disassembly,
// So we can build their CFGs.
BC->skipMarkedFragments();
BC->clearFragmentsToSkip();
+ // Function boundaries are not represented by CFG edges. Materialize an
+ // explicit tail call when a block reachable from the primary entry falls
+ // through the end of a function into the next function. The explicit
+ // transfer allows both functions to be optimized and moved independently.
+ for (auto &BFI : BC->getBinaryFunctions()) {
+ BinaryFunction &Function = BFI.second;
+ BinaryFunction *Target = getInterproceduralFallThroughTarget(*BC, Function);
+ if (!Target)
+ continue;
+
+ BC->errs() << "BOLT-WARNING: interprocedural fall-through detected from "
+ << Function.getPrintName() << " to " << Target->getPrintName()
+ << "; materializing an explicit tail call\n";
+ const BinaryBasicBlock *LastBB = Function.getLayout().block_back();
+ Function.getBasicBlockAtOffset(LastBB->getOffset())
+ ->addTailCallInstruction(Target->getSymbol());
+ }
+
BC->TotalScore = 0;
BC->SumExecutionCount = 0;
for (auto &BFI : BC->getBinaryFunctions()) {
diff --git a/bolt/test/RISCV/interprocedural-fallthrough.s b/bolt/test/RISCV/interprocedural-fallthrough.s
new file mode 100644
index 0000000000000..cda2c3047f80b
--- /dev/null
+++ b/bolt/test/RISCV/interprocedural-fallthrough.s
@@ -0,0 +1,86 @@
+// Check that BOLT materializes an explicit tail call for functions connected
+// by an implicit fall-through so they can be moved independently.
+
+// RUN: split-file %s %t.dir
+// RUN: llvm-mc -triple=riscv64 -mattr=+c -filetype=obj \
+// RUN: -o %t.dir/input.o %t.dir/input.s
+// RUN: ld.lld -q -e _start -o %t.dir/input.exe %t.dir/input.o
+// RUN: llvm-bolt %t.dir/input.exe -o %t.dir/output.exe \
+// RUN: --reorder-functions=user --function-order=%t.dir/order.txt 2>&1 \
+// RUN: | FileCheck %s --check-prefix=WARNING
+// RUN: llvm-objdump -d --no-show-raw-insn %t.dir/output.exe \
+// RUN: | FileCheck %s --check-prefix=DISASM
+// RUN: llvm-nm -n %t.dir/output.exe | FileCheck %s --check-prefix=NM
+
+// WARNING-NOT: interprocedural fall-through detected from terminated_source
+// WARNING: BOLT-WARNING: interprocedural fall-through detected from fallthrough_source to fallthrough_target; materializing an explicit tail call
+// WARNING-NOT: interprocedural fall-through detected from terminated_source
+
+// DISASM-LABEL: <fallthrough_source>:
+// DISASM: li a0, 0x0
+// DISASM-NEXT: jal {{.*}} <fallthrough_target>
+
+// NM: T fallthrough_source
+// NM-NEXT: T poison
+// NM-NEXT: T fallthrough_target
+
+//--- input.s
+ .text
+ .option rvc
+
+ .globl _start
+ .type _start, @function
+ .p2align 2
+_start:
+ call fallthrough_source
+ li a7, 93
+ ecall
+ j .
+ .size _start, .-_start
+
+ .globl fallthrough_source
+ .type fallthrough_source, @function
+ .p2align 2
+fallthrough_source:
+ li a0, 0
+ .size fallthrough_source, .-fallthrough_source
+
+ // Keep this alignment padding outside fallthrough_source's symbol size.
+ // Execution intentionally crosses it to reach fallthrough_target.
+ .p2align 2
+ .globl fallthrough_target
+ .type fallthrough_target, @function
+fallthrough_target:
+ ret
+ .size fallthrough_target, .-fallthrough_target
+
+ .globl poison
+ .type poison, @function
+ .p2align 2
+poison:
+ li a0, 1
+ ret
+ .size poison, .-poison
+
+ // The padding after this terminator must not be mistaken for a fall-through.
+ .globl terminated_source
+ .type terminated_source, @function
+ .p2align 2
+terminated_source:
+ ret
+ .size terminated_source, .-terminated_source
+
+ .p2align 2
+ .globl terminated_target
+ .type terminated_target, @function
+terminated_target:
+ ret
+ .size terminated_target, .-terminated_target
+
+//--- order.txt
+_start
+fallthrough_source
+poison
+fallthrough_target
+terminated_source
+terminated_target
diff --git a/bolt/test/X86/interprocedural-fallthrough.s b/bolt/test/X86/interprocedural-fallthrough.s
new file mode 100644
index 0000000000000..f9a6a85d46ea9
--- /dev/null
+++ b/bolt/test/X86/interprocedural-fallthrough.s
@@ -0,0 +1,88 @@
+# REQUIRES: x86_64-linux
+
+## Check that BOLT materializes an explicit tail call for functions connected
+## by an implicit fall-through so they can be moved independently.
+
+# RUN: split-file %s %t.dir
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux-gnu \
+# RUN: %t.dir/input.s -o %t.dir/input.o
+# RUN: %clang %cflags -no-pie %t.dir/input.o -o %t.dir/input.exe \
+# RUN: -Wl,--no-dynamic-linker -Wl,-q -Wl,-e,_start
+# RUN: %t.dir/input.exe
+# RUN: llvm-bolt %t.dir/input.exe -o %t.dir/output.exe \
+# RUN: --reorder-functions=user --function-order=%t.dir/order.txt 2>&1 \
+# RUN: | FileCheck %s --check-prefix=WARNING
+# RUN: %t.dir/output.exe
+# RUN: llvm-nm -n %t.dir/output.exe | FileCheck %s --check-prefix=NM
+# RUN: llvm-objdump -d --no-show-raw-insn \
+# RUN: --disassemble-symbols=fallthrough_source %t.dir/output.exe \
+# RUN: | FileCheck %s --check-prefix=DISASM
+
+# WARNING-NOT: interprocedural fall-through detected from terminated_source
+# WARNING: BOLT-WARNING: interprocedural fall-through detected from fallthrough_source to fallthrough_target; materializing an explicit tail call
+# WARNING-NOT: interprocedural fall-through detected from terminated_source
+
+# NM: T fallthrough_source
+# NM-NEXT: T poison
+# NM-NEXT: T fallthrough_target
+# DISASM: jmp {{.*}} <fallthrough_target>
+
+#--- input.s
+ .text
+
+ .globl _start
+ .type _start, @function
+ .p2align 4
+_start:
+ callq fallthrough_source
+ movq $60, %rax
+ syscall
+ ud2
+ .size _start, .-_start
+
+ .globl fallthrough_source
+ .type fallthrough_source, @function
+ .p2align 4
+fallthrough_source:
+ xorl %edi, %edi
+ .size fallthrough_source, .-fallthrough_source
+
+ # Keep this alignment padding outside fallthrough_source's symbol size.
+ # Execution intentionally crosses it to reach fallthrough_target.
+ .p2align 4
+ .globl fallthrough_target
+ .type fallthrough_target, @function
+fallthrough_target:
+ retq
+ .size fallthrough_target, .-fallthrough_target
+
+ .globl poison
+ .type poison, @function
+ .p2align 4
+poison:
+ movl $1, %edi
+ retq
+ .size poison, .-poison
+
+ # The padding after this terminator must not be mistaken for a fall-through.
+ .globl terminated_source
+ .type terminated_source, @function
+ .p2align 4
+terminated_source:
+ retq
+ .size terminated_source, .-terminated_source
+
+ .p2align 4
+ .globl terminated_target
+ .type terminated_target, @function
+terminated_target:
+ retq
+ .size terminated_target, .-terminated_target
+
+#--- order.txt
+_start
+fallthrough_source
+poison
+fallthrough_target
+terminated_source
+terminated_target
More information about the llvm-commits
mailing list