[llvm] [BOLT][AArch64] Preserve external branches to non-function symbols (PR #227275)
Alexandros Lamprineas via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 04:17:52 PDT 2026
https://github.com/labrinea created https://github.com/llvm/llvm-project/pull/227275
Record the ELF type of input relocation symbols. Branches to non-function symbols can carry live IP0/IP1, so keep them as external branches instead of converting them to tail calls. Preserve tail-call handling for PLT entries, whose undefined symbols may also have STT_NOTYPE.
Lower conditional external exits through local transfer blocks and retain their profile counts through FDATA and YAML. Experimental relaxation can then use B-only thunks without clobbering live IP0/IP1.
Make AArch64 branch analysis decline blocks ending in an external exit. Their target has no local CFG successor; reporting incomplete analysis as success caused branch fixup to add a redundant local jump after a conditional branch.
Add AArch64 tests for function-body targets, symbol aliases, conditional exits, and a split block with a local conditional followed by an external branch.
Assisted-by: Codex
>From a9293773de6a6447d3657614bedb930e11d50977 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas at arm.com>
Date: Tue, 29 Sep 2026 10:20:27 +0000
Subject: [PATCH] [BOLT][AArch64] Preserve external branches to non-function
symbols
Record the ELF type of input relocation symbols. Branches to non-function
symbols can carry live IP0/IP1, so keep them as external branches instead of
converting them to tail calls. Preserve tail-call handling for PLT entries,
whose undefined symbols may also have STT_NOTYPE.
Lower conditional external exits through local transfer blocks and retain
their profile counts through FDATA and YAML. Experimental relaxation can
then use B-only thunks without clobbering live IP0/IP1.
Make AArch64 branch analysis decline blocks ending in an external exit.
Their target has no local CFG successor; reporting incomplete analysis as
success caused branch fixup to add a redundant local jump after a conditional
branch.
Add AArch64 tests for function-body targets, symbol aliases, conditional
exits, and a split block with a local conditional followed by an external
branch.
Assisted-by: Codex
---
bolt/include/bolt/Core/BinaryFunction.h | 12 +-
bolt/include/bolt/Core/BinarySection.h | 3 +-
bolt/include/bolt/Core/MCPlus.h | 1 +
bolt/include/bolt/Core/MCPlusBuilder.h | 6 +
bolt/include/bolt/Core/Relocation.h | 12 +-
bolt/lib/Core/BinaryFunction.cpp | 168 ++++++++++--------
bolt/lib/Core/BinaryFunctionProfile.cpp | 7 +-
bolt/lib/Core/MCPlusBuilder.cpp | 8 +
bolt/lib/Passes/Inliner.cpp | 6 +
bolt/lib/Passes/LongJmp.cpp | 6 +-
bolt/lib/Profile/DataReader.cpp | 14 +-
bolt/lib/Profile/StaleProfileMatching.cpp | 5 +-
bolt/lib/Profile/YAMLProfileReader.cpp | 11 +-
bolt/lib/Profile/YAMLProfileWriter.cpp | 8 +-
bolt/lib/Rewrite/RewriteInstance.cpp | 4 +-
.../Target/AArch64/AArch64MCPlusBuilder.cpp | 10 +-
.../AArch64/branch-target-symbol-type-alias.S | 85 +++++++++
.../conditional-branch-target-symbol-type.S | 108 +++++++++++
.../external-branch-local-conditional-split.S | 44 +++++
.../tail-classified-body-branch-live-x16.S | 133 ++++++++++++++
20 files changed, 547 insertions(+), 104 deletions(-)
create mode 100644 bolt/test/AArch64/branch-target-symbol-type-alias.S
create mode 100644 bolt/test/AArch64/conditional-branch-target-symbol-type.S
create mode 100644 bolt/test/AArch64/external-branch-local-conditional-split.S
create mode 100644 bolt/test/AArch64/tail-classified-body-branch-live-x16.S
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index c41ad06027f64..4af7e21cb29de 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -44,6 +44,7 @@
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/MC/MCInst.h"
@@ -1386,7 +1387,8 @@ class BinaryFunction {
/// against \p Symbol.
/// Assert if the \p Address is not inside this function.
void addRelocation(uint64_t Address, MCSymbol *Symbol, uint32_t RelType,
- uint64_t Addend, uint64_t Value);
+ uint64_t Addend, uint64_t Value,
+ uint8_t ELFSymType = ELF::STT_NOTYPE);
/// Return locations (offsets) of data section relocations targeting internals
/// of this functions.
@@ -2475,11 +2477,9 @@ class BinaryFunction {
/// Clear execution profile of the function.
void clearProfile();
- /// Converts conditional tail calls to unconditional tail calls. We do this to
- /// handle conditional tail calls correctly and to give a chance to the
- /// simplify conditional tail call pass to decide whether to re-optimize them
- /// using profile information.
- void removeConditionalTailCalls();
+ /// Split conditional tail calls and conditional external branches into local
+ /// CFG edges to blocks with unconditional transfers.
+ void removeConditionalExits();
// Convert COUNT_NO_PROFILE to 0
void removeTagsFromProfile();
diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h
index 3e3ad8217ea71..7ee66d306cc7e 100644
--- a/bolt/include/bolt/Core/BinarySection.h
+++ b/bolt/include/bolt/Core/BinarySection.h
@@ -369,7 +369,8 @@ class BinarySection {
uint64_t Offset, MCSymbol *Symbol, uint32_t Type, uint64_t Addend,
uint64_t Value = 0, bool IsRELR = false,
uint32_t JmpRelocationIndex = Relocation::NoJmpRelocationIndex) {
- addDynamicRelocation(Relocation{Offset, Symbol, Type, Addend, Value, IsRELR,
+ addDynamicRelocation(Relocation{Offset, Symbol, Type, Addend, Value,
+ ELF::STT_NOTYPE, IsRELR,
JmpRelocationIndex});
}
diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h
index e97c31049f48d..a54fe7e08b470 100644
--- a/bolt/include/bolt/Core/MCPlus.h
+++ b/bolt/include/bolt/Core/MCPlus.h
@@ -68,6 +68,7 @@ class MCAnnotation {
kJumpTable, /// Jump Table.
kTailCall, /// Tail call.
kConditionalTailCall, /// CTC.
+ kExternalBranch, /// Branch to a target outside the current function.
kOffset, /// Offset in the function.
kLabel, /// MCSymbol pointing to this instruction.
kSize, /// Size of the instruction.
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 79298c57985f2..e561786823a03 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1370,6 +1370,12 @@ class MCPlusBuilder {
/// Return true if the instruction is a tail call.
bool isTailCall(const MCInst &Inst) const;
+ /// Mark a branch whose target is outside the current function.
+ void setExternalBranch(MCInst &Inst) const;
+
+ /// Return true if the branch target is outside the current function.
+ bool isExternalBranch(const MCInst &Inst) const;
+
/// Stores NegateRAState annotation on \p Inst.
void setNegateRAState(MCInst &Inst) const;
diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h
index 02d18ff97cf98..9b0aedb151245 100644
--- a/bolt/include/bolt/Core/Relocation.h
+++ b/bolt/include/bolt/Core/Relocation.h
@@ -14,6 +14,7 @@
#ifndef BOLT_CORE_RELOCATION_H
#define BOLT_CORE_RELOCATION_H
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/TargetParser/Triple.h"
@@ -40,11 +41,12 @@ class Relocation {
static constexpr uint32_t NoJmpRelocationIndex = (1u << 30) - 1;
Relocation(uint64_t Offset, MCSymbol *Symbol, uint32_t Type, uint64_t Addend,
- uint64_t Value, bool IsRELR = false,
+ uint64_t Value, uint8_t ELFSymType = ELF::STT_NOTYPE,
+ bool IsRELR = false,
uint32_t JmpRelocationIndex = NoJmpRelocationIndex)
: Offset(Offset), Symbol(Symbol), Addend(Addend), Value(Value),
- Type(Type), JmpRelocationIndex(JmpRelocationIndex), Optional(false),
- IsRELR(IsRELR) {
+ Type(Type), ELFSymType(ELFSymType),
+ JmpRelocationIndex(JmpRelocationIndex), Optional(false), IsRELR(IsRELR) {
assert((isRelative() || !isRELR()) &&
"Only relative relocations can be relr.");
}
@@ -72,6 +74,10 @@ class Relocation {
/// Relocation type.
uint32_t Type;
+ /// ELF type of the original input symbol. This can differ from the type of
+ /// the canonicalized MCSymbol above.
+ uint8_t ELFSymType = ELF::STT_NOTYPE;
+
private:
/// Original index in DT_JMPREL, or NoJmpRelocationIndex for relocations
/// originating from other relocation tables.
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 69eec01e098c1..24306dc4006b6 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1165,20 +1165,32 @@ MCSymbol *BinaryFunction::handleExternalReference(MCInst &Instruction,
assert(!MIB->isTailCall(Instruction) &&
"synthetic tail call instruction found");
- // This is a call regardless of the opcode.
- // Assign proper opcode for tail calls, so that they could be
- // treated as calls.
+ // AArch64 branches against non-function symbols can carry live IP0/IP1.
+ // PLT entries are an exception: they use these registers even when the
+ // relocation names an undefined STT_NOTYPE symbol.
+ const Relocation *BranchRelocation =
+ BC.isAArch64() ? getRelocationAt(Offset) : nullptr;
+ const bool IsFunction =
+ BranchRelocation && BranchRelocation->ELFSymType == ELF::STT_FUNC;
+ const BinaryFunction *TargetBF = BC.getBinaryFunctionAtAddress(TargetAddress);
+ const bool IsPLTTarget = TargetBF && TargetBF->isPLTFunction();
+
if (!IsCall) {
- if (!MIB->convertJmpToTailCall(Instruction)) {
- assert(MIB->isConditionalBranch(Instruction) &&
- "unknown tail call instruction");
- if (opts::Verbosity >= 2) {
- BC.errs() << "BOLT-WARNING: conditional tail call detected in "
- << "function " << *this << " at 0x"
- << Twine::utohexstr(AbsoluteInstrAddr) << ".\n";
+ if (BranchRelocation && !IsFunction && !IsPLTTarget) {
+ MIB->setExternalBranch(Instruction);
+ } else {
+ // Assign proper opcode for tail calls, so they can be treated as calls.
+ if (!MIB->convertJmpToTailCall(Instruction)) {
+ assert(MIB->isConditionalBranch(Instruction) &&
+ "unknown tail call instruction");
+ if (opts::Verbosity >= 2) {
+ BC.errs() << "BOLT-WARNING: conditional tail call detected in "
+ << "function " << *this << " at 0x"
+ << Twine::utohexstr(AbsoluteInstrAddr) << ".\n";
+ }
}
+ IsCall = true;
}
- IsCall = true;
}
if (opts::Verbosity >= 2 && TargetAddress == 0) {
@@ -1459,7 +1471,7 @@ Error BinaryFunction::disassemble() {
}
}
- if (!IsCall) {
+ if (!IsCall && containsAddress(TargetAddress)) {
// Add taken branch info.
TakenBranches.emplace_back(Offset, TargetAddress - getAddress());
}
@@ -2358,7 +2370,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
if (!InsertBB) {
// It must be a fallthrough or unreachable code. Create a new block unless
// we see an unconditional branch following a conditional one. The latter
- // should not be a conditional tail call.
+ // should not be a conditional tail call or external branch.
assert(PrevBB && "no previous basic block for a fall through");
MCInst *PrevInstr = PrevBB->getLastNonPseudoInstr();
assert(PrevInstr && "no previous instruction for a fall through");
@@ -2366,6 +2378,7 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
!MIB->isIndirectBranch(*PrevInstr) &&
!MIB->isUnconditionalBranch(*PrevInstr) &&
!MIB->getConditionalTailCall(*PrevInstr) &&
+ !MIB->isExternalBranch(*PrevInstr) &&
!MIB->isReturn(*PrevInstr)) {
// Temporarily restore inserter basic block.
InsertBB = PrevBB;
@@ -2467,14 +2480,12 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
"should have non-pseudo instruction in non-empty block");
if (BB->succ_size() == 0) {
- // Since there's no existing successors, we know the last instruction is
- // not a conditional branch. Thus if it's a terminator, it shouldn't be a
- // fall-through.
- //
- // Conditional tail call is a special case since we don't add a taken
- // branch successor for it.
+ // Conditional tail calls and external branches have no local taken
+ // successor, but can still fall through.
IsPrevFT = !MIB->isTerminator(*LastInstr) ||
- MIB->getConditionalTailCall(*LastInstr);
+ (MIB->isConditionalBranch(*LastInstr) &&
+ (MIB->isExternalBranch(*LastInstr) ||
+ MIB->isTailCall(*LastInstr)));
} else if (BB->succ_size() == 1) {
IsPrevFT = MIB->isConditionalBranch(*LastInstr);
} else {
@@ -2550,9 +2561,9 @@ Error BinaryFunction::buildCFG(MCPlusBuilder::AllocatorIdTy AllocatorId) {
void BinaryFunction::postProcessCFG() {
if (isSimple() && !BasicBlocks.empty()) {
- // Convert conditional tail call branches to conditional branches that jump
- // to a tail call.
- removeConditionalTailCalls();
+ // Convert conditional tail calls and conditional external branches into
+ // conditional branches to blocks with unconditional transfers.
+ removeConditionalExits();
postProcessProfile();
@@ -2596,78 +2607,87 @@ void BinaryFunction::removeTagsFromProfile() {
}
}
-void BinaryFunction::removeConditionalTailCalls() {
+void BinaryFunction::removeConditionalExits() {
// Blocks to be appended at the end.
std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks;
for (auto BBI = begin(); BBI != end(); ++BBI) {
BinaryBasicBlock &BB = *BBI;
- MCInst *CTCInstr = BB.getLastNonPseudoInstr();
- if (!CTCInstr)
+ MCInst *CondBranch = BB.getLastNonPseudoInstr();
+ if (!CondBranch || !BC.MIB->isConditionalBranch(*CondBranch))
continue;
- std::optional<uint64_t> TargetAddressOrNone =
- BC.MIB->getConditionalTailCall(*CTCInstr);
- if (!TargetAddressOrNone)
+ const bool IsTailCall = BC.MIB->isTailCall(*CondBranch);
+ const bool IsExternalBranch = BC.MIB->isExternalBranch(*CondBranch);
+
+ if (!IsTailCall && !IsExternalBranch)
continue;
- // Gather all necessary information about CTC instruction before
- // annotations are destroyed.
- const int32_t CFIStateBeforeCTC = BB.getCFIStateAtInstr(CTCInstr);
- uint64_t CTCTakenCount = BinaryBasicBlock::COUNT_NO_PROFILE;
- uint64_t CTCMispredCount = BinaryBasicBlock::COUNT_NO_PROFILE;
+ assert(IsTailCall != IsExternalBranch &&
+ "cannot be both tail call and external branch");
+
+ // Gather information before the branch annotations are destroyed.
+ const int32_t CFIStateBeforeBranch = BB.getCFIStateAtInstr(CondBranch);
+ uint64_t TakenCount = BinaryBasicBlock::COUNT_NO_PROFILE;
+ uint64_t MispredCount = BinaryBasicBlock::COUNT_NO_PROFILE;
if (hasValidProfile()) {
- CTCTakenCount = BC.MIB->getAnnotationWithDefault<uint64_t>(
- *CTCInstr, "CTCTakenCount");
- CTCMispredCount = BC.MIB->getAnnotationWithDefault<uint64_t>(
- *CTCInstr, "CTCMispredCount");
+ TakenCount = BC.MIB->getAnnotationWithDefault<uint64_t>(*CondBranch,
+ IsExternalBranch ? "Count" : "CTCTakenCount");
+ MispredCount = BC.MIB->getAnnotationWithDefault<uint64_t>(*CondBranch,
+ IsExternalBranch ? "MispredCount" : "CTCMispredCount");
}
- // Assert that the tail call does not throw.
- assert(!BC.MIB->getEHInfo(*CTCInstr) &&
- "found tail call with associated landing pad");
-
- // Create a basic block with an unconditional tail call instruction using
- // the same destination.
- const MCSymbol *CTCTargetLabel = BC.MIB->getTargetSymbol(*CTCInstr);
- assert(CTCTargetLabel && "symbol expected for conditional tail call");
- MCInst TailCallInstr;
- BC.MIB->createTailCall(TailCallInstr, CTCTargetLabel, BC.Ctx.get());
-
- // Move offset from CTCInstr to TailCallInstr.
- if (const std::optional<uint32_t> Offset = BC.MIB->getOffset(*CTCInstr)) {
- BC.MIB->setOffset(TailCallInstr, *Offset);
- BC.MIB->clearOffset(*CTCInstr);
+ assert(!BC.MIB->getEHInfo(*CondBranch) &&
+ "found conditional exit with associated landing pad");
+
+ // Create an unconditional transfer to the same destination.
+ const MCSymbol *TargetLabel = BC.MIB->getTargetSymbol(*CondBranch);
+ assert(TargetLabel && "symbol expected for conditional exit");
+ MCInst TransferInstr;
+ if (IsExternalBranch) {
+ BC.MIB->createUncondBranch(TransferInstr, TargetLabel, BC.Ctx.get());
+ BC.MIB->setExternalBranch(TransferInstr);
+ } else {
+ BC.MIB->createTailCall(TransferInstr, TargetLabel, BC.Ctx.get());
}
- // Link new BBs to the original input offset of the BB where the CTC
- // is, so we can map samples recorded in new BBs back to the original BB
- // seem in the input binary (if using BAT)
- std::unique_ptr<BinaryBasicBlock> TailCallBB =
- createBasicBlock(BC.Ctx->createNamedTempSymbol("TC"));
- TailCallBB->setOffset(BB.getInputOffset());
- TailCallBB->addInstruction(TailCallInstr);
- TailCallBB->setCFIState(CFIStateBeforeCTC);
+ // Move offset from the conditional branch to the unconditional transfer.
+ if (const std::optional<uint32_t> Offset = BC.MIB->getOffset(*CondBranch)) {
+ BC.MIB->setOffset(TransferInstr, *Offset);
+ BC.MIB->clearOffset(*CondBranch);
+ }
- // Add CFG edge with profile info from BB to TailCallBB.
- BB.addSuccessor(TailCallBB.get(), CTCTakenCount, CTCMispredCount);
+ // Link the new block to the input offset for BAT sample mapping.
+ std::unique_ptr<BinaryBasicBlock> TransferBB = createBasicBlock(
+ BC.Ctx->createNamedTempSymbol(IsExternalBranch ? "EB" : "TC"));
+ TransferBB->setOffset(BB.getInputOffset());
+ TransferBB->addInstruction(TransferInstr);
+ TransferBB->setCFIState(CFIStateBeforeBranch);
+
+ // Add CFG edge with profile info from BB to TransferBB.
+ BB.addSuccessor(TransferBB.get(), TakenCount, MispredCount);
// Add execution count for the block.
- TailCallBB->setExecutionCount(CTCTakenCount);
+ TransferBB->setExecutionCount(TakenCount);
- BC.MIB->convertTailCallToJmp(*CTCInstr);
+ if (IsExternalBranch)
+ BC.MIB->removeAnnotation(*CondBranch,
+ MCPlus::MCAnnotation::kExternalBranch);
+ else
+ BC.MIB->convertTailCallToJmp(*CondBranch);
- BC.MIB->replaceBranchTarget(*CTCInstr, TailCallBB->getLabel(),
+ BC.MIB->replaceBranchTarget(*CondBranch, TransferBB->getLabel(),
BC.Ctx.get());
- // Add basic block to the list that will be added to the end.
- NewBlocks.emplace_back(std::move(TailCallBB));
+ // Add the transfer block to the end of the function.
+ NewBlocks.emplace_back(std::move(TransferBB));
- // Swap edges as the TailCallBB corresponds to the taken branch.
+ // The new block corresponds to the taken branch.
BB.swapConditionalSuccessors();
- // This branch is no longer a conditional tail call.
- BC.MIB->unsetConditionalTailCall(*CTCInstr);
+ // X86's convertTailCallToJmp leaves conditional branches unchanged.
+ if (IsTailCall)
+ BC.MIB->unsetConditionalTailCall(*CondBranch);
}
insertBasicBlocks(std::prev(end()), std::move(NewBlocks),
@@ -4878,7 +4898,8 @@ bool BinaryFunction::isPossibleVeneer() const {
void BinaryFunction::addRelocation(uint64_t Address, MCSymbol *Symbol,
uint32_t RelType, uint64_t Addend,
- uint64_t Value) {
+ uint64_t Value,
+ uint8_t ELFSymType) {
assert(Address >= getAddress() && Address < getAddress() + getMaxSize() &&
"address is outside of the function");
uint64_t Offset = Address - getAddress();
@@ -4889,7 +4910,8 @@ void BinaryFunction::addRelocation(uint64_t Address, MCSymbol *Symbol,
std::map<uint64_t, Relocation> &Rels =
IsCI ? Islands->Relocations : Relocations;
if (BC.MIB->shouldRecordCodeRelocation(RelType))
- Rels[Offset] = Relocation{Offset, Symbol, RelType, Addend, Value};
+ Rels[Offset] = Relocation{Offset, Symbol, RelType, Addend, Value,
+ ELFSymType};
}
} // namespace bolt
diff --git a/bolt/lib/Core/BinaryFunctionProfile.cpp b/bolt/lib/Core/BinaryFunctionProfile.cpp
index aed94e8a80073..7cc58646978e7 100644
--- a/bolt/lib/Core/BinaryFunctionProfile.cpp
+++ b/bolt/lib/Core/BinaryFunctionProfile.cpp
@@ -132,9 +132,8 @@ void BinaryFunction::postProcessProfile() {
++SuccBIIter;
}
- // Set the execution count of the basic block to be the maximum execution
- // count across the indirect branches, indirect calls and call
- // instructions. All other instructions can be ignored.
+ // Set the execution count of the basic block to the maximum count across
+ // indirect branches, calls and external branches.
uint64_t MaxCount = BB->getExecutionCount();
for (MCInst &Inst : *BB) {
uint64_t ExecCount = 0;
@@ -144,7 +143,7 @@ void BinaryFunction::postProcessProfile() {
for (IndirectCallProfile &Entry : *ICSP)
ExecCount += Entry.Count;
}
- } else if (BC.MIB->isCall(Inst)) {
+ } else if (BC.MIB->isCall(Inst) || BC.MIB->isExternalBranch(Inst)) {
if (auto Count = BC.MIB->tryGetAnnotationAs<uint64_t>(Inst, "Count"))
ExecCount = *Count;
}
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index 1bcae94214909..89545ec901f36 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -159,6 +159,14 @@ bool MCPlusBuilder::isTailCall(const MCInst &Inst) const {
return false;
}
+void MCPlusBuilder::setExternalBranch(MCInst &Inst) const {
+ setAnnotationOpValue(Inst, MCAnnotation::kExternalBranch, true);
+}
+
+bool MCPlusBuilder::isExternalBranch(const MCInst &Inst) const {
+ return hasAnnotation(Inst, MCAnnotation::kExternalBranch);
+}
+
void MCPlusBuilder::setNegateRAState(MCInst &Inst) const {
assert(!hasAnnotation(Inst, MCAnnotation::kNegateState));
setAnnotationOpValue(Inst, MCAnnotation::kNegateState, true);
diff --git a/bolt/lib/Passes/Inliner.cpp b/bolt/lib/Passes/Inliner.cpp
index 775e4a36f39ea..c776ef689ae73 100644
--- a/bolt/lib/Passes/Inliner.cpp
+++ b/bolt/lib/Passes/Inliner.cpp
@@ -161,6 +161,12 @@ InliningInfo getInliningInfo(const BinaryFunction &BF) {
bool HasCFI = false;
bool IsLeaf = true;
+ // Inlining assumes every non-tail-call branch targets a block in the callee.
+ for (const BinaryBasicBlock &BB : BF)
+ for (const MCInst &Inst : BB)
+ if (BC.MIB->isExternalBranch(Inst))
+ return INL_NONE;
+
// Perform necessary checks unless the option overrides it.
if (!opts::mustConsider(BF)) {
if (BF.hasSDTMarker())
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 3dab645cfd61a..2d8325528bc5f 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -821,10 +821,12 @@ bool LongJmpPass::relaxLocalBranches(BinaryFunction &BF,
continue;
const MCSymbol *TargetSymbol = MIB->getTargetSymbol(*Inst);
- BB->eraseInstruction(BB->findInstruction(Inst));
-
BinaryBasicBlock::BinaryBranchInfo BI;
BinaryBasicBlock *TargetBB = BB->getSuccessor(TargetSymbol, BI);
+ if (!TargetBB)
+ continue;
+
+ BB->eraseInstruction(BB->findInstruction(Inst));
// Erasing the unconditional branch shrinks BB by one instruction.
BinaryBasicBlock *TrampolineBB =
diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp
index 7fcaffd763ad3..95384c6fc42f5 100644
--- a/bolt/lib/Profile/DataReader.cpp
+++ b/bolt/lib/Profile/DataReader.cpp
@@ -634,13 +634,13 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
if (!FBD)
return;
- // Profile information for calls.
+ // Profile information for transfers out of a function.
//
// There are 3 cases that we annotate differently:
- // 1) Conditional tail calls that could be mispredicted.
+ // 1) Conditional tail calls and conditional external branches.
// 2) Indirect calls to multiple destinations with mispredictions.
// Before we validate CFG we have to handle indirect branches here too.
- // 3) Regular direct calls. The count could be different from containing
+ // 3) Other direct transfers. Their count could differ from the containing
// basic block count. Keep this data in case we find it useful.
//
for (BranchInfo &BI : FBD->Data) {
@@ -649,8 +649,9 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
continue;
MCInst *Instr = BF.getInstructionAtOffset(BI.From.Offset);
- if (!Instr ||
- (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr)))
+ if (!Instr || (!BC.MIB->isCall(*Instr) &&
+ !BC.MIB->isIndirectBranch(*Instr) &&
+ !BC.MIB->isExternalBranch(*Instr)))
continue;
auto setOrUpdateAnnotation = [&](StringRef Name, uint64_t Count) {
@@ -677,6 +678,9 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
setOrUpdateAnnotation("CTCMispredCount", BI.Mispreds);
} else {
setOrUpdateAnnotation("Count", BI.Branches);
+ if (BC.MIB->isExternalBranch(*Instr) &&
+ BC.MIB->isConditionalBranch(*Instr))
+ setOrUpdateAnnotation("MispredCount", BI.Mispreds);
}
}
}
diff --git a/bolt/lib/Profile/StaleProfileMatching.cpp b/bolt/lib/Profile/StaleProfileMatching.cpp
index 90562b63aed1f..f09d4ba0da047 100644
--- a/bolt/lib/Profile/StaleProfileMatching.cpp
+++ b/bolt/lib/Profile/StaleProfileMatching.cpp
@@ -1043,8 +1043,11 @@ void assignProfile(BinaryFunction &BF,
// is executed; conservatively, setting it to the count of the block
setOrUpdateAnnotation(Instr, "CTCTakenCount", Block.Flow);
BC.MIB->removeAnnotation(Instr, "CTCMispredCount");
- } else if (BC.MIB->isCall(Instr)) {
+ } else if (BC.MIB->isCall(Instr) || BC.MIB->isExternalBranch(Instr)) {
setOrUpdateAnnotation(Instr, "Count", Block.Flow);
+ if (BC.MIB->isExternalBranch(Instr) &&
+ BC.MIB->isConditionalBranch(Instr))
+ BC.MIB->removeAnnotation(Instr, "MispredCount");
}
}
}
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 6df2e39957937..fddfeb6e7d9b6 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -70,7 +70,7 @@ void YAMLProfileReader::CallGraphMatcher::constructBFCG(
for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
for (const BinaryBasicBlock &BB : BF->blocks()) {
for (const MCInst &Instr : BB) {
- if (!BC.MIB->isCall(Instr))
+ if (!BC.MIB->isCall(Instr) && !BC.MIB->isExternalBranch(Instr))
continue;
const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
if (!CallSymbol)
@@ -268,9 +268,11 @@ bool YAMLProfileReader::parseFunctionProfile(
++MismatchedCalls;
continue;
}
- if (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr)) {
+ if (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr) &&
+ !BC.MIB->isExternalBranch(*Instr)) {
if (opts::Verbosity >= 2)
- errs() << "BOLT-WARNING: expected call at offset " << YamlCSI.Offset
+ errs() << "BOLT-WARNING: expected call or external branch at offset "
+ << YamlCSI.Offset
<< " in block " << BB.getName() << '\n';
++MismatchedCalls;
continue;
@@ -296,6 +298,9 @@ bool YAMLProfileReader::parseFunctionProfile(
setAnnotation("CTCMispredCount", YamlCSI.Mispreds);
} else {
setAnnotation("Count", YamlCSI.Count);
+ if (BC.MIB->isExternalBranch(*Instr) &&
+ BC.MIB->isConditionalBranch(*Instr))
+ setAnnotation("MispredCount", YamlCSI.Mispreds);
}
}
diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp
index 9d6a3983c3d03..6ef36974f86d3 100644
--- a/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -267,7 +267,8 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
YamlBB.ExecCount = BB->getKnownExecutionCount();
for (const MCInst &Instr : *BB) {
- if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr))
+ if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr) &&
+ !BC.MIB->isExternalBranch(Instr))
continue;
SmallVector<std::pair<StringRef, yaml::bolt::CallSiteInfo>> CSTargets;
@@ -292,7 +293,7 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
CSI.Mispreds = CSP.Mispreds;
CSTargets.emplace_back(TargetName, CSI);
}
- } else { // direct call or a tail call
+ } else { // direct call, tail call, or external branch
StringRef TargetName = "";
const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Instr);
const BinaryFunction *const Callee =
@@ -308,6 +309,9 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
CSI.Mispreds = getAnnotationWithDefault(Instr, "CTCMispredCount");
} else {
CSI.Count = getAnnotationWithDefault(Instr, "Count");
+ if (BC.MIB->isExternalBranch(Instr) &&
+ BC.MIB->isConditionalBranch(Instr))
+ CSI.Mispreds = getAnnotationWithDefault(Instr, "MispredCount");
}
if (CSI.Count)
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 49472928233bb..bf186733e3be0 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -3351,8 +3351,10 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};
symbol_iterator SymbolIter = Rel.getSymbol();
+ uint8_t ELFSymType = ELF::STT_NOTYPE;
if (SymbolIter != InputFile->symbol_end()) {
SymbolRef Symbol = *SymbolIter;
+ ELFSymType = ELFSymbolRef(Symbol).getELFType();
section_iterator Section =
cantFail(Symbol.getSection(), "cannot get symbol section");
if (Section != InputFile->section_end()) {
@@ -3639,7 +3641,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
if (IsFromCode)
ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType,
- Addend, ExtractedValue);
+ Addend, ExtractedValue, ELFSymType);
else if (IsToCode || ForceRelocation)
BC->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, Addend,
ExtractedValue);
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index bd4fcd308c254..ec8fef5b8caeb 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -2732,7 +2732,11 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
if (isPseudo(*I) || isNoop(*I))
continue;
- // Stop when we find the first non-terminator
+ // An external exit cannot be rewritten using local CFG successors.
+ if (isExternalBranch(*I))
+ return false;
+
+ // Stop at instructions not represented by a branch in the local CFG.
if (!isTerminator(*I) || isTailCall(*I) || !isBranch(*I))
break;
@@ -3044,11 +3048,11 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
case ELF::R_AARCH64_PREL16:
case ELF::R_AARCH64_PREL32:
case ELF::R_AARCH64_PREL64:
- return true;
- case ELF::R_AARCH64_CALL26:
case ELF::R_AARCH64_JUMP26:
case ELF::R_AARCH64_TSTBR14:
case ELF::R_AARCH64_CONDBR19:
+ return true;
+ case ELF::R_AARCH64_CALL26:
case ELF::R_AARCH64_TLSDESC_CALL:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
diff --git a/bolt/test/AArch64/branch-target-symbol-type-alias.S b/bolt/test/AArch64/branch-target-symbol-type-alias.S
new file mode 100644
index 0000000000000..d680534eb3316
--- /dev/null
+++ b/bolt/test/AArch64/branch-target-symbol-type-alias.S
@@ -0,0 +1,85 @@
+## Two symbols at the same address give different permissions to their branch
+## relocations. The branch to the NOTYPE symbol must preserve x16; the branch
+## to the FUNC symbol may use an x16-clobbering veneer.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --lite=0 --relax-exp \
+# RUN: --max-thunk-chain-length=0 --reorder-functions=exec-count \
+# RUN: | FileCheck %s --check-prefix=BOLT
+# RUN: llvm-objdump -d --disassemble-symbols=foo,baz,__AArch64_forward_Thunk_0,__AArch64_forward_ADRPThunk_body_func_0 \
+# RUN: %t.bolt | FileCheck %s --check-prefix=DISASM
+
+# BOLT: relaxed 1 calls with long thunks
+# BOLT: 1 long thunks created
+# BOLT: relaxed 1 unconditional branches
+# BOLT: 1 branch thunks created
+
+# DISASM: <foo>:
+# DISASM: mov x16, #0x2a
+# DISASM-NEXT: b {{.*}} <__AArch64_forward_Thunk_0>
+# DISASM: <baz>:
+# DISASM: b {{.*}} <__AArch64_forward_ADRPThunk_body_func_0>
+# DISASM: <__AArch64_forward_ADRPThunk_body_func_0>:
+# DISASM-NEXT: adrp x16,
+# DISASM-NEXT: add x16, x16,
+# DISASM-NEXT: br x16
+# DISASM: <__AArch64_forward_Thunk_0>:
+# DISASM-NEXT: b {{.*}} <body_func>
+
+ .text
+ .globl bar
+ .type bar, %function
+bar:
+ b .bar_ret
+ .globl body_notype
+ .type body_notype, %notype
+body_notype:
+ .globl body_func
+ .type body_func, %function
+body_func:
+ cbnz x0, .bar_ret
+ cmp x16, #42
+ cset w0, ne
+ ret
+.bar_ret:
+ ret
+ .size bar, .-bar
+
+ .globl foo
+ .type foo, %function
+foo:
+# FDATA: 1 foo #foo# 100
+ mov x0, #0
+ mov x16, #42
+ b body_notype
+ .size foo, .-foo
+
+ .globl baz
+ .type baz, %function
+baz:
+# FDATA: 1 baz #baz# 100
+ mov x0, #1
+ b body_func
+ .size baz, .-baz
+
+ .globl padding1
+ .type padding1, %function
+padding1:
+# FDATA: 1 padding1 #padding1# 100
+ ret
+ .space 0x4400000
+ .size padding1, .-padding1
+
+ .globl padding2
+ .type padding2, %function
+padding2:
+# FDATA: 1 padding2 #padding2# 100
+ ret
+ .space 0x4400000
+ .size padding2, .-padding2
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/conditional-branch-target-symbol-type.S b/bolt/test/AArch64/conditional-branch-target-symbol-type.S
new file mode 100644
index 0000000000000..713b4627de499
--- /dev/null
+++ b/bolt/test/AArch64/conditional-branch-target-symbol-type.S
@@ -0,0 +1,108 @@
+## Conditional branches to a NOTYPE symbol in another function can carry live
+## x16. They must reach the target through B-only branch thunks. A FUNC alias
+## at the same address can still use a long call thunk.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --lite=0 --relax-exp \
+# RUN: --max-thunk-chain-length=0 --reorder-functions=exec-count \
+# RUN: | FileCheck %s --check-prefix=BOLT
+# RUN: llvm-objdump -d --disassemble-symbols=foo,baz,__AArch64_forward_Thunk_0,__AArch64_forward_ADRPThunk_body_func_0 \
+# RUN: %t.bolt | FileCheck %s --check-prefix=DISASM
+# RUN: link_fdata %s %t %t.lbr.fdata FDATA-LBR
+# RUN: llvm-bolt %t -o %t.lbr.bolt --data %t.lbr.fdata -w %t.yaml \
+# RUN: --lite=0 --relax-exp --print-cfg | FileCheck %s --check-prefix=PROFILE
+# RUN: llvm-bolt %t -o %t.yaml.bolt --data %t.yaml \
+# RUN: --lite=0 --relax-exp --print-cfg | FileCheck %s --check-prefix=PROFILE
+
+# FDATA-LBR: 0 [unknown] 0 1 foo 0 0 100
+# FDATA-LBR: 1 foo 8 1 body_notype 0 2 60
+# FDATA-LBR: 1 foo c 1 body_notype 0 3 30
+
+# PROFILE-LABEL: Binary Function "foo" after building cfg {
+# PROFILE: Exec Count : 100
+# PROFILE: cbz x2, {{.*}} # Count: 60 # MispredCount: 2
+# PROFILE: Successors: {{.*}} (mispreds: 2, count: 60)
+# PROFILE: tbz w3, #0x0, {{.*}} # Count: 30 # MispredCount: 3
+# PROFILE: Successors: {{.*}} (mispreds: 3, count: 30)
+
+# BOLT: relaxed 1 calls with long thunks
+# BOLT: relaxed 2 unconditional branches
+# BOLT: 1 branch thunks created
+
+# DISASM: <foo>:
+# DISASM: mov x16, #0x2a
+# DISASM: cbz x2, 0x[[EB1:[0-9a-f]+]] <foo+{{.*}}>
+# DISASM-NEXT: tbz w3, #0x0, 0x[[EB2:[0-9a-f]+]] <foo+{{.*}}>
+# DISASM-NEXT: ret
+# DISASM-NEXT: [[EB1]]: {{.*}} b {{.*}} <__AArch64_forward_Thunk_0>
+# DISASM-NEXT: [[EB2]]: {{.*}} b {{.*}} <__AArch64_forward_Thunk_0>
+# DISASM: <baz>:
+# DISASM: cbz x2, {{.*}} <baz+{{.*}}>
+# DISASM: b {{.*}} <__AArch64_forward_ADRPThunk_body_func_0>
+# DISASM: <__AArch64_forward_ADRPThunk_body_func_0>:
+# DISASM-NEXT: adrp x16,
+# DISASM-NEXT: add x16, x16,
+# DISASM-NEXT: br x16
+# DISASM: <__AArch64_forward_Thunk_0>:
+# DISASM-NEXT: b {{.*}} <body_func>
+
+ .text
+ .globl bar
+ .type bar, %function
+bar:
+ b .bar_ret
+ .globl body_notype
+ .type body_notype, %notype
+body_notype:
+ .globl body_func
+ .type body_func, %function
+body_func:
+ cbnz x0, .bar_ret
+ cmp x16, #42
+ cset w0, ne
+ ret
+.bar_ret:
+ ret
+ .size bar, .-bar
+
+ .globl foo
+ .type foo, %function
+foo:
+# FDATA: 1 foo #foo# 100
+ mov x0, #0
+ mov x16, #42
+ cbz x2, body_notype
+ tbz x3, #0, body_notype
+ ret
+ .size foo, .-foo
+
+ .globl baz
+ .type baz, %function
+baz:
+# FDATA: 1 baz #baz# 100
+ mov x0, #1
+ cbz x2, body_func
+ ret
+ .size baz, .-baz
+
+ .globl padding1
+ .type padding1, %function
+padding1:
+# FDATA: 1 padding1 #padding1# 100
+ ret
+ .space 0x4400000
+ .size padding1, .-padding1
+
+ .globl padding2
+ .type padding2, %function
+padding2:
+# FDATA: 1 padding2 #padding2# 100
+ ret
+ .space 0x4400000
+ .size padding2, .-padding2
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/external-branch-local-conditional-split.S b/bolt/test/AArch64/external-branch-local-conditional-split.S
new file mode 100644
index 0000000000000..3b0c9e3fe0396
--- /dev/null
+++ b/bolt/test/AArch64/external-branch-local-conditional-split.S
@@ -0,0 +1,44 @@
+## A local conditional branch can share a block with a following external
+## branch. Branch fixup must not add a redundant local branch after that exit.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t -nostdlib
+# RUN: link_fdata --no-lbr %s %t %t.fdata
+# RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --lite=0 --relax-exp \
+# RUN: --split-functions --split-strategy=all --print-after-branch-fixup \
+# RUN: --print-only=foo | FileCheck %s
+
+# CHECK-LABEL: Binary Function "foo" after fix-branches {
+# CHECK: IsSplit : 1
+# CHECK: b.eq
+# CHECK-NEXT: {{.*}}b{{[[:space:]]}}
+# CHECK-NEXT: Successors:
+# CHECK: BOLT-INFO: starting experimental relaxation pass
+
+ .text
+ .globl bar
+ .type bar, %function
+bar:
+ b .Lbar_ret
+ .globl bar_body
+ .type bar_body, %notype
+bar_body:
+ ret
+.Lbar_ret:
+ ret
+ .size bar, .-bar
+
+ .globl foo
+ .type foo, %function
+foo:
+# FDATA: 1 foo #foo# 100
+ cmp x0, #0
+ b.eq .Lfoo_ret
+ b bar_body
+.Lfoo_ret:
+ ret
+ .size foo, .-foo
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/tail-classified-body-branch-live-x16.S b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
new file mode 100644
index 0000000000000..53e04beb04a28
--- /dev/null
+++ b/bolt/test/AArch64/tail-classified-body-branch-live-x16.S
@@ -0,0 +1,133 @@
+## Check that a direct B to a function-body symbol is not relaxed as an
+## ABI tail call. Such a branch can carry live caller-saved registers, so
+## relaxation must use B-only branch chains instead of ADRP/BR call thunks.
+##
+## In the TAILCALL run, bar_body has FUNC type, so the branch may use an ABI
+## tail-call thunk. In the BRANCH run, bar_body has NOTYPE and must be relaxed
+## as a branch even though it has the same address.
+
+# REQUIRES: system-linux
+
+# RUN: %clang %cflags -Wl,-q -Wl,-e,foo %s -o %t.tailcall -nostdlib
+# RUN: link_fdata --no-lbr %s %t.tailcall %t.tailcall.fdata
+# RUN: llvm-bolt %t.tailcall -o %t.tailcall.bolt --data %t.tailcall.fdata \
+# RUN: --relax-exp --reorder-functions=exec-count --skip-funcs='^bar$' \
+# RUN: | FileCheck %s --check-prefix=TAILCALL-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=foo,bar,bar_body,padding,__AArch64_forward_ADRPThunk_bar_body_0 \
+# RUN: %t.tailcall.bolt | FileCheck %s --check-prefix=TAILCALL
+
+# RUN: %clang %cflags -DBRANCH -Wl,-q -Wl,-e,foo %s -o %t.branch -nostdlib
+# RUN: link_fdata --no-lbr %s %t.branch %t.branch.fdata
+# RUN: llvm-bolt %t.branch -o %t.branch.bolt --data %t.branch.fdata \
+# RUN: --lite=0 --relax-exp --reorder-functions=exec-count \
+# RUN: | FileCheck %s --check-prefix=BRANCH-BOLT
+# RUN: llvm-objdump -d \
+# RUN: --disassemble-symbols=foo,bar,bar_body,.bar_ret,padding,__AArch64_forward_Thunk_0 \
+# RUN: %t.branch.bolt | FileCheck %s --check-prefix=BRANCH
+
+# TAILCALL-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
+# TAILCALL-BOLT: BOLT-INFO: starting experimental relaxation pass
+# TAILCALL-BOLT: BOLT-INFO: relaxed 1 calls with long thunks
+# TAILCALL-BOLT: BOLT-INFO: 1 long thunks created
+
+# BRANCH-BOLT: BOLT-INFO: relaxing branches for compact code model (<128MB)
+# BRANCH-BOLT: BOLT-INFO: starting experimental relaxation pass
+# BRANCH-BOLT: BOLT-INFO: relaxed 1 unconditional branches
+# BRANCH-BOLT: BOLT-INFO: 1 branch thunks created
+
+ .text
+ .globl bar
+ .type bar, %function
+bar:
+.bar_entry:
+ b .bar_ret
+ .globl bar_body
+#ifdef BRANCH
+ .type bar_body, %notype
+#else
+ .type bar_body, %function
+#endif
+bar_body:
+ cmp x16, #42
+ cset w0, ne
+ ret
+.bar_ret:
+ ret
+ .size bar, .-bar
+
+ .globl foo
+ .type foo, %function
+foo:
+.foo_entry:
+# FDATA: 1 foo #.foo_entry# 100
+ mov x16, #42
+ b bar_body
+ .space 0x4400000
+ .size foo, .-foo
+
+ .globl padding
+ .type padding, %function
+padding:
+.padding_entry:
+# FDATA: 1 padding #.padding_entry# 100
+ ret
+ .space 0x4400000
+ .size padding, .-padding
+
+## Force relocation mode.
+ .reloc 0, R_AARCH64_NONE
+
+# TAILCALL: Disassembly of section .bolt.org.text:
+#
+# TAILCALL: <bar>:
+# TAILCALL-NEXT: {{.*}} b {{.*}} <.bar_ret>
+# TAILCALL: <bar_body>:
+# TAILCALL-NEXT: {{.*}} cmp x16, #0x2a
+# TAILCALL-NEXT: {{.*}} cset w0, ne
+# TAILCALL-NEXT: {{.*}} ret
+#
+# TAILCALL: Disassembly of section .text:
+#
+# TAILCALL: <foo>:
+# TAILCALL-NEXT: {{.*}} mov x16, #0x2a
+# TAILCALL-NEXT: {{.*}} b {{.*}} <__AArch64_forward_ADRPThunk_bar_body_0>
+#
+# TAILCALL: <__AArch64_forward_ADRPThunk_bar_body_0>:
+# TAILCALL-NEXT: {{.*}} adrp x16,
+# TAILCALL-NEXT: {{.*}} add x16, x16,
+# TAILCALL-NEXT: {{.*}} br x16
+#
+# TAILCALL: <padding>:
+# TAILCALL-NEXT: {{.*}} ret
+
+
+# BRANCH: Disassembly of section .bolt.org.text:
+#
+# BRANCH: <bar_body>:
+# BRANCH-NEXT: {{.*}} cmp x16, #0x2a
+# BRANCH-NEXT: {{.*}} cset w0, ne
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: Disassembly of section .text:
+#
+# BRANCH: <foo>:
+# BRANCH-NEXT: {{.*}} mov x16, #0x2a
+# BRANCH-NEXT: {{.*}} b 0x[[CHAIN:[0-9a-f]+]] <__AArch64_forward_Thunk_0>
+#
+# BRANCH: <__AArch64_forward_Thunk_0>:
+# BRANCH-NEXT: [[CHAIN]]: {{.*}} b 0x[[BODY:[0-9a-f]+]] <bar+0x4>
+#
+# BRANCH: <padding>:
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: Disassembly of section .text.cold:
+#
+# BRANCH: <bar>:
+# BRANCH-NEXT: {{.*}} b 0x[[RET:[0-9a-f]+]] <.bar_ret>
+# BRANCH-NEXT: [[BODY]]: {{.*}} cmp x16, #0x2a
+# BRANCH-NEXT: {{.*}} cset w0, ne
+# BRANCH-NEXT: {{.*}} ret
+#
+# BRANCH: <.bar_ret>:
+# BRANCH-NEXT: [[RET]]: {{.*}} ret
More information about the llvm-commits
mailing list