[lld] [lld][macho] Track max thunks to create and remove --slop_scale (PR #193372)
Ellis Hoag via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 12:00:05 PDT 2026
https://github.com/ellishg updated https://github.com/llvm/llvm-project/pull/193372
>From 2ee3fb1be03ab27c2eacc2682ee42248adbc4772 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Fri, 17 Apr 2026 17:06:05 -0700
Subject: [PATCH 01/17] [lld][macho] Restructure thunk generation alg
---
lld/MachO/ConcatOutputSection.cpp | 425 +++++++++++++-----------------
lld/MachO/ConcatOutputSection.h | 3 -
lld/test/MachO/arm64-thunks.s | 20 +-
3 files changed, 187 insertions(+), 261 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 753dea90d0f5d..5585a17c5c071 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -15,6 +15,7 @@
#include "Target.h"
#include "lld/Common/CommonLinkerContext.h"
#include "llvm/BinaryFormat/MachO.h"
+#include <deque>
using namespace llvm;
using namespace llvm::MachO;
@@ -144,115 +145,11 @@ bool TextOutputSection::needsThunks() const {
std::min(target->backwardBranchRange, target->forwardBranchRange))
return false;
// Yes, this program is large enough to need thunks.
- if (parent) {
+ if (parent)
parent->needsThunks = true;
- }
- for (ConcatInputSection *isec : inputs) {
- for (Relocation &r : isec->relocs) {
- if (!target->hasAttr(r.type, RelocAttrBits::BRANCH))
- continue;
- auto *sym = cast<Symbol *>(r.referent);
- // Pre-populate the thunkMap and memoize call site counts for every
- // InputSection and ThunkInfo. We do this for the benefit of
- // estimateBranchTargetThresholdVA().
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{sym, r.addend}];
- // Knowing ThunkInfo call site count will help us know whether or not we
- // might need to create more for this referent at the time we are
- // estimating distance to __stubs in estimateBranchTargetThresholdVA().
- ++thunkInfo.callSiteCount;
- // We can avoid work on InputSections that have no BRANCH relocs.
- isec->hasCallSites = true;
- }
- }
return true;
}
-// Estimate the address beyond which branch targets (like __stubs and
-// __objc_stubs) are within range of a simple forward branch. This is called
-// exactly once, when the last input section has been finalized.
-uint64_t
-TextOutputSection::estimateBranchTargetThresholdVA(size_t callIdx) const {
- // Tally the functions which still have call sites remaining to process,
- // which yields the maximum number of thunks we might yet place.
- size_t maxPotentialThunks = 0;
- for (auto &tp : thunkMap) {
- ThunkInfo &ti = tp.second;
- // This overcounts: Only sections that are in forward jump range from the
- // currently-active section get finalized, and all input sections are
- // finalized when estimateBranchTargetThresholdVA() is called. So only
- // backward jumps will need thunks, but we count all jumps.
- if (ti.callSitesUsed < ti.callSiteCount)
- maxPotentialThunks += 1;
- }
- // Tally the total size of input sections remaining to process.
- uint64_t isecVA = inputs[callIdx]->getVA();
- uint64_t isecEnd = isecVA;
- for (size_t i = callIdx; i < inputs.size(); i++) {
- InputSection *isec = inputs[i];
- isecEnd = alignToPowerOf2(isecEnd, isec->align) + isec->getSize();
- }
-
- // Tally up any thunks that have already been placed that have VA higher than
- // inputs[callIdx]. First, find the index of the first thunk that is beyond
- // the current inputs[callIdx].
- auto itPostcallIdxThunks =
- llvm::partition_point(thunks, [isecVA](const ConcatInputSection *t) {
- return t->getVA() <= isecVA;
- });
- uint64_t existingForwardThunks = thunks.end() - itPostcallIdxThunks;
-
- uint64_t forwardBranchRange = target->forwardBranchRange;
- assert(isecEnd > forwardBranchRange &&
- "should not run thunk insertion if all code fits in jump range");
- assert(isecEnd - isecVA <= forwardBranchRange &&
- "should only finalize sections in jump range");
-
- // Estimate the maximum size of the code, right before the branch target
- // sections.
- uint64_t maxTextSize = 0;
- // Add the size of all the inputs, including the unprocessed ones.
- maxTextSize += isecEnd;
-
- // Add the size of the thunks that have already been created that are ahead of
- // inputs[callIdx]. These are already created thunks that will be interleaved
- // with inputs[callIdx...end].
- maxTextSize += existingForwardThunks * target->thunkSize;
-
- // Add the size of the thunks that may be created in the future. Since
- // 'maxPotentialThunks' overcounts, this is an estimate of the upper limit.
- maxTextSize += maxPotentialThunks * target->thunkSize;
-
- // Calculate the total size of all late branch target sections
- uint64_t branchTargetsSize = 0;
-
- // Add the size of __stubs section
- branchTargetsSize += in.stubs->getSize();
-
- // Add the size of __objc_stubs section if it exists
- if (in.objcStubs && in.objcStubs->isNeeded())
- branchTargetsSize += in.objcStubs->getSize();
-
- // Estimated maximum VA of the last branch target.
- uint64_t maxVAOfLastBranchTarget = maxTextSize + branchTargetsSize;
-
- // Estimate the address after which call sites can safely call branch targets
- // directly rather than through intermediary thunks.
- uint64_t branchTargetThresholdVA =
- maxVAOfLastBranchTarget - forwardBranchRange;
-
- log("thunks = " + std::to_string(thunkMap.size()) +
- ", potential = " + std::to_string(maxPotentialThunks) +
- ", stubs = " + std::to_string(in.stubs->getSize()) +
- (in.objcStubs && in.objcStubs->isNeeded()
- ? ", objc_stubs = " + std::to_string(in.objcStubs->getSize())
- : "") +
- ", isecVA = " + utohexstr(isecVA) + ", threshold = " +
- utohexstr(branchTargetThresholdVA) + ", isecEnd = " + utohexstr(isecEnd) +
- ", tail = " + utohexstr(isecEnd - isecVA) +
- ", slop = " + utohexstr(forwardBranchRange - (isecEnd - isecVA)));
- return branchTargetThresholdVA;
-}
-
void ConcatOutputSection::finalizeOne(ConcatInputSection *isec) {
size = alignToPowerOf2(size, isec->align);
fileSize = alignToPowerOf2(fileSize, isec->align);
@@ -276,171 +173,203 @@ void TextOutputSection::finalize() {
uint64_t forwardBranchRange = target->forwardBranchRange;
uint64_t backwardBranchRange = target->backwardBranchRange;
- uint64_t branchTargetThresholdVA = TargetInfo::outOfRangeVA;
size_t thunkSize = target->thunkSize;
- size_t relocCount = 0;
- size_t callSiteCount = 0;
- size_t thunkCallCount = 0;
size_t thunkCount = 0;
-
- // Walk all sections in order. Finalize all sections that are less than
- // forwardBranchRange in front of it.
- // isecVA is the address of the current section.
- // addr + size is the start address of the first non-finalized section.
-
- // inputs[finalIdx] is for finalization (address-assignment)
- size_t finalIdx = 0;
- // Kick-off by ensuring that the first input section has an address
- for (size_t callIdx = 0, endIdx = inputs.size(); callIdx < endIdx;
- ++callIdx) {
- if (finalIdx == callIdx)
- finalizeOne(inputs[finalIdx++]);
- ConcatInputSection *isec = inputs[callIdx];
- assert(isec->isFinal);
- uint64_t isecVA = isec->getVA();
-
- // Assign addresses up-to the forward branch-range limit.
- // Every call instruction needs a small number of bytes (on Arm64: 4),
- // and each inserted thunk needs a slightly larger number of bytes
- // (on Arm64: 12). If a section starts with a branch instruction and
- // contains several branch instructions in succession, then the distance
- // from the current position to the position where the thunks are inserted
- // grows. So leave room for a bunch of thunks.
- unsigned slop = config->slopScale * thunkSize;
- while (finalIdx < endIdx) {
- uint64_t expectedNewSize =
- alignToPowerOf2(addr + size, inputs[finalIdx]->align) +
- inputs[finalIdx]->getSize();
- if (expectedNewSize >= isecVA + forwardBranchRange - slop)
+ size_t thunkCallCount = 0;
+ DenseSet<uint64_t> residentThunkPages;
+
+ auto isTargetInRange = [&](const ConcatInputSection &isec,
+ const Relocation &r) -> bool {
+ uint64_t callVA = isec.getVA() + r.offset;
+ uint64_t lowVA =
+ backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
+ uint64_t highVA = callVA + forwardBranchRange;
+ auto *funcSym = cast<Symbol *>(r.referent);
+ uint64_t funcVA = resolveSymbolOffsetVA(funcSym, r.type, r.addend);
+ // Check if the referent is reachable with a simple call instruction.
+ return lowVA <= funcVA && funcVA <= highVA;
+ };
+ auto getThunkInRange = [&](const ConcatInputSection &isec,
+ const Relocation &r) -> Defined * {
+ assert(!isTargetInRange(isec, r));
+ uint64_t callVA = isec.getVA() + r.offset;
+ uint64_t lowVA =
+ backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
+ uint64_t highVA = callVA + forwardBranchRange;
+ auto *funcSym = cast<Symbol *>(r.referent);
+ ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
+ if (thunkInfo.sym) {
+ uint64_t thunkVA = thunkInfo.isec->getVA();
+ if (lowVA <= thunkVA && thunkVA <= highVA)
+ return thunkInfo.sym;
+ }
+ return nullptr;
+ };
+ auto createThunk = [&](const ConcatInputSection &isec, Relocation &r) {
+ assert(!isTargetInRange(isec, r));
+ assert(getThunkInRange(isec, r) == nullptr);
+ assert(isec.isFinal);
+ uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
+ if (addr + size > highVA) {
+ // There were too many consecutive branch instructions for `slop`
+ // above. If you hit this: For the current algorithm, just bumping up
+ // slop above and trying again is probably simplest. (See also PR51578
+ // comment 5).
+ fatal(Twine(__FUNCTION__) +
+ ": FIXME: thunk range overrun. Consider increasing the "
+ "slop-scale with `--slop-scale=<unsigned_int>`.");
+ }
+ auto *funcSym = cast<Symbol *>(r.referent);
+ ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
+ thunkInfo.isec =
+ makeSyntheticInputSection(isec.getSegName(), isec.getName());
+ thunkInfo.isec->parent = this;
+ assert(thunkInfo.isec->live);
+
+ std::string addendSuffix;
+ if (r.addend != 0)
+ addendSuffix = "+" + std::to_string(r.addend);
+ StringRef thunkName =
+ saver().save(funcSym->getName() + addendSuffix + ".thunk." +
+ std::to_string(thunkInfo.sequence++));
+ if (!isa<Defined>(funcSym) || cast<Defined>(funcSym)->isExternal()) {
+ r.referent = thunkInfo.sym = symtab->addDefined(
+ thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
+ /*isWeakDef=*/false, /*isPrivateExtern=*/true,
+ /*isReferencedDynamically=*/false, /*noDeadStrip=*/false,
+ /*isWeakDefCanBeHidden=*/false);
+ } else {
+ r.referent = thunkInfo.sym = make<Defined>(
+ thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
+ /*isWeakDef=*/false, /*isExternal=*/false, /*isPrivateExtern=*/true,
+ /*includeInSymtab=*/true, /*isReferencedDynamically=*/false,
+ /*noDeadStrip=*/false, /*isWeakDefCanBeHidden=*/false);
+ }
+ thunkInfo.sym->used = true;
+ target->populateThunk(thunkInfo.isec, funcSym, r.addend);
+ // The thunk itself bakes in the addend, so the call-site reloc must
+ // branch to the thunk start with no extra offset.
+ r.addend = 0;
+ finalizeOne(thunkInfo.isec);
+ thunks.push_back(thunkInfo.isec);
+ uint64_t thunkFirstByte = thunkInfo.isec->getVA();
+ uint64_t thunkLastByte = thunkFirstByte + thunkInfo.isec->getSize() - 1;
+ residentThunkPages.insert(thunkFirstByte / target->getPageSize());
+ residentThunkPages.insert(thunkLastByte / target->getPageSize());
+ ++thunkCount;
+ ++thunkCallCount;
+ };
+
+ std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
+ branchesToProcess;
+ SmallVector<std::tuple<ConcatInputSection *, Relocation *, Defined *>>
+ deferredBranchRedirects;
+
+ const uint64_t slop = config->slopScale * thunkSize;
+ for (auto *isec : inputs) {
+ while (!branchesToProcess.empty()) {
+ auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
+ assert(callerIsec->isFinal);
+ if (isTargetInRange(*callerIsec, *r)) {
+ branchesToProcess.pop_front();
+ continue;
+ }
+ if (auto *sym = getThunkInRange(*callerIsec, *r)) {
+ deferredBranchRedirects.emplace_back(callerIsec, r, sym);
+ branchesToProcess.pop_front();
+ continue;
+ }
+ uint64_t highVA = callerIsec->getVA() + r->offset + forwardBranchRange;
+ uint64_t nextEnd =
+ alignToPowerOf2(addr + size, isec->align) + isec->getSize();
+ if (nextEnd + slop <= highVA)
break;
- finalizeOne(inputs[finalIdx++]);
+
+ createThunk(*callerIsec, *r);
+ branchesToProcess.pop_front();
}
+ finalizeOne(isec);
- if (!isec->hasCallSites)
+ bool hasCallsite = llvm::any_of(isec->relocs, [](Relocation &r) {
+ return target->hasAttr(r.type, RelocAttrBits::BRANCH);
+ });
+ if (!hasCallsite)
continue;
- if (finalIdx == endIdx &&
- branchTargetThresholdVA == TargetInfo::outOfRangeVA) {
- // When we have finalized all input sections, branch target sections (like
- // __stubs and __objc_stubs) (destined to follow __text) come within range
- // of forward branches and we can estimate the threshold address after
- // which we can reach any branch target with a forward branch. Note that
- // although it sits in the middle of a loop, this code executes only once.
- // It is in the loop because we need to call it at the proper
- // time: the earliest call site from which the end of __text
- // (and start of branch target sections) comes within range of a forward
- // branch.
- branchTargetThresholdVA = estimateBranchTargetThresholdVA(callIdx);
- }
// Process relocs by ascending address, i.e., ascending offset within isec
- std::vector<Relocation> &relocs = isec->relocs;
// FIXME: This property does not hold for object files produced by ld64's
// `-r` mode.
- assert(is_sorted(relocs, [](Relocation &a, Relocation &b) {
+ assert(is_sorted(isec->relocs, [](Relocation &a, Relocation &b) {
return a.offset > b.offset;
}));
- for (Relocation &r : reverse(relocs)) {
- ++relocCount;
+ for (Relocation &r : reverse(isec->relocs)) {
if (!target->hasAttr(r.type, RelocAttrBits::BRANCH))
continue;
- ++callSiteCount;
- // Calculate branch reachability boundaries
- uint64_t callVA = isecVA + r.offset;
- uint64_t lowVA =
- backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
- uint64_t highVA = callVA + forwardBranchRange;
- // Calculate our call referent address
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
- // The referent is not reachable, so we need to use a thunk... unless we
- // are close enough to the end that branch target sections (__stubs,
- // __objc_stubs) are now within range of a simple forward branch -- BUT
- // only for zero-addend branches. The writer's resolveSymbolOffsetVA()
- // resolves non-zero-addend branches against the symbol body rather than
- // the stub, so __stubs reachability says nothing about whether such a
- // call can be emitted directly. Hence the `r.addend == 0` guard below.
- // See INTERP check lines in arm64-thunk-branch-addend.s.
- if (r.addend == 0 &&
- (funcSym->isInStubs() ||
- (in.objcStubs && in.objcStubs->isNeeded() &&
- ObjCStubsSection::isObjCStubSymbol(funcSym))) &&
- callVA >= branchTargetThresholdVA) {
- assert(callVA != TargetInfo::outOfRangeVA);
+ if (isTargetInRange(*isec, r))
continue;
- }
- // Use the same resolution rules as the writer: for non-zero addends this
- // goes directly to the symbol body rather than any stub trampoline.
- // See INTERP check lines in arm64-thunk-branch-addend.s.
- uint64_t funcVA = resolveSymbolOffsetVA(funcSym, r.type, r.addend);
- ++thunkInfo.callSitesUsed;
- if (lowVA <= funcVA && funcVA <= highVA) {
- // The referent is reachable with a simple call instruction.
+ if (auto *sym = getThunkInRange(*isec, r)) {
+ deferredBranchRedirects.emplace_back(isec, &r, sym);
continue;
}
- ++thunkInfo.thunkCallCount;
- ++thunkCallCount;
- // If an existing thunk is reachable, use it ...
- if (thunkInfo.sym) {
- uint64_t thunkVA = thunkInfo.isec->getVA();
- if (lowVA <= thunkVA && thunkVA <= highVA) {
- r.referent = thunkInfo.sym;
- // The thunk itself bakes in the addend, so the call-site reloc must
- // branch to the thunk start with no extra offset.
- r.addend = 0;
- continue;
- }
- }
- // ... otherwise, create a new thunk.
- if (addr + size > highVA) {
- // There were too many consecutive branch instructions for `slop`
- // above. If you hit this: For the current algorithm, just bumping up
- // slop above and trying again is probably simplest. (See also PR51578
- // comment 5).
- fatal(Twine(__FUNCTION__) +
- ": FIXME: thunk range overrun. Consider increasing the "
- "slop-scale with `--slop-scale=<unsigned_int>`.");
- }
- thunkInfo.isec =
- makeSyntheticInputSection(isec->getSegName(), isec->getName());
- thunkInfo.isec->parent = this;
- assert(thunkInfo.isec->live);
-
- std::string addendSuffix;
- if (r.addend != 0)
- addendSuffix = "+" + std::to_string(r.addend);
- StringRef thunkName =
- saver().save(funcSym->getName() + addendSuffix + ".thunk." +
- std::to_string(thunkInfo.sequence++));
- if (!isa<Defined>(funcSym) || cast<Defined>(funcSym)->isExternal()) {
- r.referent = thunkInfo.sym = symtab->addDefined(
- thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
- /*isWeakDef=*/false, /*isPrivateExtern=*/true,
- /*isReferencedDynamically=*/false, /*noDeadStrip=*/false,
- /*isWeakDefCanBeHidden=*/false);
- } else {
- r.referent = thunkInfo.sym = make<Defined>(
- thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
- /*isWeakDef=*/false, /*isExternal=*/false, /*isPrivateExtern=*/true,
- /*includeInSymtab=*/true, /*isReferencedDynamically=*/false,
- /*noDeadStrip=*/false, /*isWeakDefCanBeHidden=*/false);
- }
- thunkInfo.sym->used = true;
- target->populateThunk(thunkInfo.isec, funcSym, r.addend);
+ auto *funcSym = cast<Symbol *>(r.referent);
+ ThunkKey key{funcSym, r.addend};
+ branchesToProcess.emplace_back(isec, &r, key);
+ }
+ }
+ for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
+ if (isTargetInRange(*callerIsec, *r))
+ continue;
+ r->referent = thunk;
+ r->addend = 0;
+ ++thunkCallCount;
+ }
+ deferredBranchRedirects.clear();
+
+ llvm::erase_if(branchesToProcess, [&](auto &tuple) {
+ auto [callerIsec, r, thunkKey] = tuple;
+ return isTargetInRange(*callerIsec, *r);
+ });
+ // Count the number of new thunk we will need to create
+ DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
+ for (auto [callerIsec, r, thunkKey] : branchesToProcess)
+ if (!getThunkInRange(*callerIsec, *r))
+ branchTargets.insert(thunkKey);
+
+ uint64_t estimatedTextEnd = addr + size + branchTargets.size() * thunkSize;
+ uint64_t estimatedStubsEnd =
+ alignToPowerOf2(estimatedTextEnd, in.stubs->align) + in.stubs->getSize();
+ if (in.objcStubs && in.objcStubs->isNeeded())
+ estimatedStubsEnd =
+ alignToPowerOf2(estimatedStubsEnd, in.objcStubs->align) +
+ in.objcStubs->getSize();
+
+ for (auto [isec, r, thunkKey] : branchesToProcess) {
+ uint64_t highVA = isec->getVA() + r->offset + forwardBranchRange;
+ auto *funcSym = cast<Symbol *>(r->referent);
+ if ((funcSym->isInStubs() ||
+ (in.objcStubs && in.objcStubs->isNeeded() &&
+ ObjCStubsSection::isObjCStubSymbol(funcSym))) &&
+ r->addend == 0 && estimatedStubsEnd <= highVA) {
+ // The branch target can reach any section in __stubs or __objc_stubs
+ assert(isec->getVA() != TargetInfo::outOfRangeVA);
+ continue;
+ }
+ if (auto *sym = getThunkInRange(*isec, *r)) {
+ r->referent = sym;
// The thunk itself bakes in the addend, so the call-site reloc must
// branch to the thunk start with no extra offset.
- r.addend = 0;
- finalizeOne(thunkInfo.isec);
- thunks.push_back(thunkInfo.isec);
- ++thunkCount;
+ r->addend = 0;
+ ++thunkCallCount;
+ continue;
}
+ createThunk(*isec, *r);
}
- log("thunks for " + parent->name + "," + name +
- ": funcs = " + std::to_string(thunkMap.size()) +
- ", relocs = " + std::to_string(relocCount) +
- ", all calls = " + std::to_string(callSiteCount) +
- ", thunk calls = " + std::to_string(thunkCallCount) +
- ", thunks = " + std::to_string(thunkCount));
+ if (thunkCount)
+ log("Created " + Twine(thunkCount) + " (" +
+ Twine(thunkCount * thunkSize / 1024) + " KB) thunks residing in " +
+ Twine(residentThunkPages.size()) + " pages and updated " +
+ Twine(thunkCallCount) + " branch targets");
}
void ConcatOutputSection::writeTo(uint8_t *buf) const {
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index dd27743199649..04fd501a9c2cd 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -100,9 +100,6 @@ struct ThunkInfo {
ConcatInputSection *isec = nullptr; // input section for active thunk
// The following values are cumulative across all thunks on this function
- uint32_t callSiteCount = 0; // how many calls to the real function?
- uint32_t callSitesUsed = 0; // how many call sites processed so-far?
- uint32_t thunkCallCount = 0; // how many call sites went to thunk?
uint8_t sequence = 0; // how many thunks created so-far?
};
diff --git a/lld/test/MachO/arm64-thunks.s b/lld/test/MachO/arm64-thunks.s
index cd3842895c47e..f482f01a690d4 100644
--- a/lld/test/MachO/arm64-thunks.s
+++ b/lld/test/MachO/arm64-thunks.s
@@ -84,7 +84,7 @@
# CHECK: bl 0x[[#%x, A]] <_a>
# CHECK: bl 0x[[#%x, B]] <_b>
# CHECK: bl 0x[[#%x, C]] <_c>
-# CHECK: bl 0x[[#%x, D_THUNK_0]] <_d.thunk.0>
+# CHECK: bl 0x[[#%x, D:]] <_d>
# CHECK: bl 0x[[#%x, E_THUNK_0]] <_e.thunk.0>
# CHECK: bl 0x[[#%x, F_THUNK_0]] <_f.thunk.0>
# CHECK: bl 0x[[#%x, G_THUNK_0]] <_g.thunk.0>
@@ -95,9 +95,9 @@
# CHECK: bl 0x[[#%x, A]] <_a>
# CHECK: bl 0x[[#%x, B]] <_b>
# CHECK: bl 0x[[#%x, C]] <_c>
-# CHECK: bl 0x[[#%x, D:]] <_d>
+# CHECK: bl 0x[[#%x, D]] <_d>
# CHECK: bl 0x[[#%x, E:]] <_e>
-# CHECK: bl 0x[[#%x, F_THUNK_0]] <_f.thunk.0>
+# CHECK: bl 0x[[#%x, F:]] <_f>
# CHECK: bl 0x[[#%x, G_THUNK_0]] <_g.thunk.0>
# CHECK: bl 0x[[#%x, H_THUNK_0]] <_h.thunk.0>
# CHECK: bl 0x[[#%x, NAN_THUNK_0]] <___nan.thunk.0>
@@ -132,8 +132,8 @@
# CHECK: bl 0x[[#%x, C]] <_c>
# CHECK: bl 0x[[#%x, D]] <_d>
# CHECK: bl 0x[[#%x, E]] <_e>
-# CHECK: bl 0x[[#%x, F_THUNK_0]] <_f.thunk.0>
-# CHECK: bl 0x[[#%x, G_THUNK_0]] <_g.thunk.0>
+# CHECK: bl 0x[[#%x, F]] <_f>
+# CHECK: bl 0x[[#%x, G:]] <_g>
# CHECK: bl 0x[[#%x, H_THUNK_0]] <_h.thunk.0>
# CHECK: bl 0x[[#%x, NAN_THUNK_0]] <___nan.thunk.0>
@@ -143,9 +143,9 @@
# CHECK: bl 0x[[#%x, C]] <_c>
# CHECK: bl 0x[[#%x, D]] <_d>
# CHECK: bl 0x[[#%x, E]] <_e>
-# CHECK: bl 0x[[#%x, F:]] <_f>
-# CHECK: bl 0x[[#%x, G:]] <_g>
-# CHECK: bl 0x[[#%x, H_THUNK_0]] <_h.thunk.0>
+# CHECK: bl 0x[[#%x, F]] <_f>
+# CHECK: bl 0x[[#%x, G]] <_g>
+# CHECK: bl 0x[[#%x, H:]] <_h>
# CHECK: bl 0x[[#%x, NAN_THUNK_0]] <___nan.thunk.0>
# CHECK: [[#%x, F_PAGE + F_OFFSET]] <_f>:
@@ -156,7 +156,7 @@
# CHECK: bl 0x[[#%x, E]] <_e>
# CHECK: bl 0x[[#%x, F]] <_f>
# CHECK: bl 0x[[#%x, G]] <_g>
-# CHECK: bl 0x[[#%x, H_THUNK_0]] <_h.thunk.0>
+# CHECK: bl 0x[[#%x, H]] <_h>
# CHECK: bl 0x[[#%x, NAN_THUNK_0]] <___nan.thunk.0>
# CHECK: [[#%x, G_PAGE + G_OFFSET]] <_g>:
@@ -167,7 +167,7 @@
# CHECK: bl 0x[[#%x, E]] <_e>
# CHECK: bl 0x[[#%x, F]] <_f>
# CHECK: bl 0x[[#%x, G]] <_g>
-# CHECK: bl 0x[[#%x, H:]] <_h>
+# CHECK: bl 0x[[#%x, H]] <_h>
# CHECK: bl 0x[[#%x, STUBS:]]
# CHECK: [[#%x, A_THUNK_0]] <_a.thunk.0>:
>From fcbba6ce88b65435c3b3ad76b81b69c63b4411ec Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 21 Apr 2026 11:19:53 -0700
Subject: [PATCH 02/17] Remove outdated comment
---
lld/MachO/ConcatOutputSection.cpp | 58 -------------------------------
1 file changed, 58 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 5585a17c5c071..79a04e4161a4d 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -54,64 +54,6 @@ void ConcatOutputSection::addInput(ConcatInputSection *input) {
// The optimal approach is to mix islands for destinations within two hops,
// and use thunks for destinations at greater distance. For now, we only
// implement thunks. TODO: Adding support for branch islands!
-//
-// Internally -- as expressed in LLD's data structures -- a
-// branch-range-extension thunk consists of:
-//
-// (1) new Defined symbol for the thunk named
-// <FUNCTION>.thunk.<SEQUENCE>, which references ...
-// (2) new InputSection, which contains ...
-// (3.1) new data for the instructions to load & branch to the far address +
-// (3.2) new Relocs on instructions to load the far address, which reference ...
-// (4.1) existing Defined symbol for the real function in __text, or
-// (4.2) existing DylibSymbol for the real function in a dylib
-//
-// Nearly-optimal thunk-placement algorithm features:
-//
-// * Single pass: O(n) on the number of call sites.
-//
-// * Accounts for the exact space overhead of thunks - no heuristics
-//
-// * Exploits the full range of call instructions - forward & backward
-//
-// Data:
-//
-// * DenseMap<ThunkKey, ThunkInfo> thunkMap: Maps each (referent, addend)
-// pair seen on a branch relocation to its thunk bookkeeper.
-//
-// * struct ThunkInfo (bookkeeper): Call instructions have limited range, and
-// distant call sites might be unable to reach the same thunk, so multiple
-// thunks are necessary to serve all call sites in a very large program. A
-// thunkInfo stores state for all thunks associated with a particular
-// function:
-// (a) thunk symbol
-// (b) input section containing stub code, and
-// (c) sequence number for the active thunk incarnation.
-// When an old thunk goes out of range, we increment the sequence number and
-// create a new thunk named <FUNCTION>.thunk.<SEQUENCE>.
-//
-// * A thunk consists of
-// (a) a Defined symbol pointing to
-// (b) an InputSection holding machine code (similar to a MachO stub), and
-// (c) relocs referencing the real function for fixing up the stub code.
-//
-// * std::vector<InputSection *> MergedInputSection::thunks: A vector parallel
-// to the inputs vector. We store new thunks via cheap vector append, rather
-// than costly insertion into the inputs vector.
-//
-// Control Flow:
-//
-// * During address assignment, MergedInputSection::finalize() examines call
-// sites by ascending address and creates thunks. When a function is beyond
-// the range of a call site, we need a thunk. Place it at the largest
-// available forward address from the call site. Call sites increase
-// monotonically and thunks are always placed as far forward as possible;
-// thus, we place thunks at monotonically increasing addresses. Once a thunk
-// is placed, it and all previous input-section addresses are final.
-//
-// * ConcatInputSection::finalize() and ConcatInputSection::writeTo() merge
-// the inputs and thunks vectors (both ordered by ascending address), which
-// is simple and cheap.
DenseMap<ThunkKey, ThunkInfo, ThunkMapKeyInfo> lld::macho::thunkMap;
>From 2699b90ec71b8d6dbe512193c4b968f8f984de34 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 21 Apr 2026 14:22:06 -0700
Subject: [PATCH 03/17] minor cleanup
---
lld/MachO/ConcatOutputSection.cpp | 9 ++++-----
lld/MachO/ConcatOutputSection.h | 2 +-
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 79a04e4161a4d..25345ee879737 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -116,8 +116,8 @@ void TextOutputSection::finalize() {
uint64_t forwardBranchRange = target->forwardBranchRange;
uint64_t backwardBranchRange = target->backwardBranchRange;
size_t thunkSize = target->thunkSize;
- size_t thunkCount = 0;
size_t thunkCallCount = 0;
+ size_t thunkCount = 0;
DenseSet<uint64_t> residentThunkPages;
auto isTargetInRange = [&](const ConcatInputSection &isec,
@@ -148,13 +148,12 @@ void TextOutputSection::finalize() {
return nullptr;
};
auto createThunk = [&](const ConcatInputSection &isec, Relocation &r) {
- assert(!isTargetInRange(isec, r));
assert(getThunkInRange(isec, r) == nullptr);
assert(isec.isFinal);
uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
if (addr + size > highVA) {
// There were too many consecutive branch instructions for `slop`
- // above. If you hit this: For the current algorithm, just bumping up
+ // below. If you hit this: For the current algorithm, just bumping up
// slop above and trying again is probably simplest. (See also PR51578
// comment 5).
fatal(Twine(__FUNCTION__) +
@@ -192,6 +191,7 @@ void TextOutputSection::finalize() {
// The thunk itself bakes in the addend, so the call-site reloc must
// branch to the thunk start with no extra offset.
r.addend = 0;
+ ++thunkCallCount;
finalizeOne(thunkInfo.isec);
thunks.push_back(thunkInfo.isec);
uint64_t thunkFirstByte = thunkInfo.isec->getVA();
@@ -199,7 +199,6 @@ void TextOutputSection::finalize() {
residentThunkPages.insert(thunkFirstByte / target->getPageSize());
residentThunkPages.insert(thunkLastByte / target->getPageSize());
++thunkCount;
- ++thunkCallCount;
};
std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
@@ -271,7 +270,7 @@ void TextOutputSection::finalize() {
auto [callerIsec, r, thunkKey] = tuple;
return isTargetInRange(*callerIsec, *r);
});
- // Count the number of new thunk we will need to create
+ // Count the number of new thunks we will need to create
DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
for (auto [callerIsec, r, thunkKey] : branchesToProcess)
if (!getThunkInRange(*callerIsec, *r))
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index 04fd501a9c2cd..154272392928a 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -99,7 +99,7 @@ struct ThunkInfo {
Defined *sym = nullptr; // private-extern symbol for active thunk
ConcatInputSection *isec = nullptr; // input section for active thunk
- // The following values are cumulative across all thunks on this function
+ // The following value is cumulative across all thunks on this function
uint8_t sequence = 0; // how many thunks created so-far?
};
>From 3e8cdeed9138fd3ef394ae545c4f1c60540e2d83 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 21 Apr 2026 17:47:51 -0700
Subject: [PATCH 04/17] remove page logs
---
lld/MachO/ConcatOutputSection.cpp | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 25345ee879737..b312bf403152d 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -118,7 +118,6 @@ void TextOutputSection::finalize() {
size_t thunkSize = target->thunkSize;
size_t thunkCallCount = 0;
size_t thunkCount = 0;
- DenseSet<uint64_t> residentThunkPages;
auto isTargetInRange = [&](const ConcatInputSection &isec,
const Relocation &r) -> bool {
@@ -194,10 +193,6 @@ void TextOutputSection::finalize() {
++thunkCallCount;
finalizeOne(thunkInfo.isec);
thunks.push_back(thunkInfo.isec);
- uint64_t thunkFirstByte = thunkInfo.isec->getVA();
- uint64_t thunkLastByte = thunkFirstByte + thunkInfo.isec->getSize() - 1;
- residentThunkPages.insert(thunkFirstByte / target->getPageSize());
- residentThunkPages.insert(thunkLastByte / target->getPageSize());
++thunkCount;
};
@@ -308,8 +303,7 @@ void TextOutputSection::finalize() {
if (thunkCount)
log("Created " + Twine(thunkCount) + " (" +
- Twine(thunkCount * thunkSize / 1024) + " KB) thunks residing in " +
- Twine(residentThunkPages.size()) + " pages and updated " +
+ Twine(thunkCount * thunkSize / 1024) + " KB) thunks and updated " +
Twine(thunkCallCount) + " branch targets");
}
>From 3a2c2421d5494fc83f28d7079257817f14fcfb7c Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Mon, 20 Apr 2026 21:10:43 -0700
Subject: [PATCH 05/17] [lld][macho] Remove --slop_scale flag
---
lld/MachO/ConcatOutputSection.cpp | 44 ++++++++++++++++++++++++-------
lld/MachO/ConcatOutputSection.h | 2 ++
lld/MachO/Config.h | 1 -
lld/MachO/Driver.cpp | 8 ------
lld/MachO/Options.td | 12 +++------
lld/test/MachO/set-slop-scale.s | 11 --------
6 files changed, 40 insertions(+), 38 deletions(-)
delete mode 100644 lld/test/MachO/set-slop-scale.s
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index b312bf403152d..d9ad94b57acc2 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -151,13 +151,13 @@ void TextOutputSection::finalize() {
assert(isec.isFinal);
uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
if (addr + size > highVA) {
- // There were too many consecutive branch instructions for `slop`
- // below. If you hit this: For the current algorithm, just bumping up
- // slop above and trying again is probably simplest. (See also PR51578
- // comment 5).
- fatal(Twine(__FUNCTION__) +
- ": FIXME: thunk range overrun. Consider increasing the "
- "slop-scale with `--slop-scale=<unsigned_int>`.");
+ // We can only encounter this if we have a massive section (> ~128MB) or
+ // an enormous number of branch instructions within a single section
+ // (> ~16M), neither of which is feasible in practice. To fix we could
+ // implement branch islands when the available space for thunks become too
+ // small.
+ fatal("encountered a branch whose target is out of range, but there is "
+ "no more space for a new thunk");
}
auto *funcSym = cast<Symbol *>(r.referent);
ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
@@ -200,17 +200,23 @@ void TextOutputSection::finalize() {
branchesToProcess;
SmallVector<std::tuple<ConcatInputSection *, Relocation *, Defined *>>
deferredBranchRedirects;
+ unsigned numPendingThunkTargets = 0;
- const uint64_t slop = config->slopScale * thunkSize;
for (auto *isec : inputs) {
while (!branchesToProcess.empty()) {
auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
assert(callerIsec->isFinal);
+ auto &thunkInfo = thunkMap[thunkKey];
if (isTargetInRange(*callerIsec, *r)) {
+ if (thunkInfo.pendingBranches.erase(r))
+ if (thunkInfo.pendingBranches.empty())
+ --numPendingThunkTargets;
branchesToProcess.pop_front();
continue;
}
if (auto *sym = getThunkInRange(*callerIsec, *r)) {
+ // The pending thunk target was already decremented when we created the
+ // thunk
deferredBranchRedirects.emplace_back(callerIsec, r, sym);
branchesToProcess.pop_front();
continue;
@@ -218,9 +224,11 @@ void TextOutputSection::finalize() {
uint64_t highVA = callerIsec->getVA() + r->offset + forwardBranchRange;
uint64_t nextEnd =
alignToPowerOf2(addr + size, isec->align) + isec->getSize();
- if (nextEnd + slop <= highVA)
+ if (nextEnd + numPendingThunkTargets * thunkSize <= highVA)
break;
+ thunkInfo.pendingBranches.clear();
+ --numPendingThunkTargets;
createThunk(*callerIsec, *r);
branchesToProcess.pop_front();
}
@@ -250,6 +258,10 @@ void TextOutputSection::finalize() {
auto *funcSym = cast<Symbol *>(r.referent);
ThunkKey key{funcSym, r.addend};
branchesToProcess.emplace_back(isec, &r, key);
+ auto &thunkInfo = thunkMap[key];
+ if (thunkInfo.pendingBranches.empty())
+ ++numPendingThunkTargets;
+ thunkInfo.pendingBranches.insert(&r);
}
}
for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
@@ -263,6 +275,12 @@ void TextOutputSection::finalize() {
llvm::erase_if(branchesToProcess, [&](auto &tuple) {
auto [callerIsec, r, thunkKey] = tuple;
+ auto &thunkInfo = thunkMap[thunkKey];
+ if (isTargetInRange(*callerIsec, *r) || getThunkInRange(*callerIsec, *r)) {
+ if (thunkInfo.pendingBranches.erase(r))
+ if (thunkInfo.pendingBranches.empty())
+ --numPendingThunkTargets;
+ }
return isTargetInRange(*callerIsec, *r);
});
// Count the number of new thunks we will need to create
@@ -270,8 +288,9 @@ void TextOutputSection::finalize() {
for (auto [callerIsec, r, thunkKey] : branchesToProcess)
if (!getThunkInRange(*callerIsec, *r))
branchTargets.insert(thunkKey);
+ assert(numPendingThunkTargets == branchTargets.size());
- uint64_t estimatedTextEnd = addr + size + branchTargets.size() * thunkSize;
+ uint64_t estimatedTextEnd = addr + size + numPendingThunkTargets * thunkSize;
uint64_t estimatedStubsEnd =
alignToPowerOf2(estimatedTextEnd, in.stubs->align) + in.stubs->getSize();
if (in.objcStubs && in.objcStubs->isNeeded())
@@ -280,6 +299,10 @@ void TextOutputSection::finalize() {
in.objcStubs->getSize();
for (auto [isec, r, thunkKey] : branchesToProcess) {
+ auto &thunkInfo = thunkMap[thunkKey];
+ if (thunkInfo.pendingBranches.erase(r))
+ if (thunkInfo.pendingBranches.empty())
+ --numPendingThunkTargets;
uint64_t highVA = isec->getVA() + r->offset + forwardBranchRange;
auto *funcSym = cast<Symbol *>(r->referent);
if ((funcSym->isInStubs() ||
@@ -300,6 +323,7 @@ void TextOutputSection::finalize() {
}
createThunk(*isec, *r);
}
+ assert(numPendingThunkTargets == 0);
if (thunkCount)
log("Created " + Twine(thunkCount) + " (" +
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index 154272392928a..90012fed980b3 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -99,6 +99,8 @@ struct ThunkInfo {
Defined *sym = nullptr; // private-extern symbol for active thunk
ConcatInputSection *isec = nullptr; // input section for active thunk
+ llvm::DenseSet<const Relocation *> pendingBranches;
+
// The following value is cumulative across all thunks on this function
uint8_t sequence = 0; // how many thunks created so-far?
};
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 9767cc5e5b6e4..9d3be9c17a39c 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -226,7 +226,6 @@ struct Configuration {
bool disableVerify;
bool separateCstringLiteralSections;
bool tailMergeStrings;
- unsigned slopScale = 256;
bool callGraphProfileSort = false;
llvm::StringRef printSymbolOrder;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 26b39f7a28d0d..9d8cbb43b64ea 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -2015,14 +2015,6 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
OPT_no_separate_cstring_literal_sections, false);
config->tailMergeStrings =
args.hasFlag(OPT_tail_merge_strings, OPT_no_tail_merge_strings, false);
- if (auto *arg = args.getLastArg(OPT_slop_scale_eq)) {
- StringRef v(arg->getValue());
- unsigned slop = 0;
- if (!llvm::to_integer(v, slop))
- error(arg->getSpelling() +
- ": expected a non-negative integer, but got '" + v + "'");
- config->slopScale = slop;
- }
auto IncompatWithCGSort = [&](StringRef firstArgStr) {
// Throw an error only if --call-graph-profile-sort is explicitly specified
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index b7686d66a258e..fd67be034b60d 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -1105,14 +1105,10 @@ defm tail_merge_strings
: BB<"tail-merge-strings", "Enable string tail merging",
"Disable string tail merging to improve link-time performance">,
Group<grp_rare>;
-def slop_scale_eq
- : Joined<["--"], "slop_scale=">,
- MetaVarName<"<unsigned_int>">,
- HelpText<"Specify the slop scale. Default value is 256. If your binary "
- "has too many consecutive branch instructions resulting in "
- "thunk-range overrun, then you need to increase this value to a "
- "higher value, such as 512 or 1024, etc">,
- Group<grp_rare>;
+def slop_scale_eq : Joined<["--"], "slop_scale=">,
+ MetaVarName<"<unsigned_int>">,
+ HelpText<"Deprecated. Has no effect">,
+ Group<grp_rare>;
def grp_deprecated : OptionGroup<"deprecated">, HelpText<"DEPRECATED">;
diff --git a/lld/test/MachO/set-slop-scale.s b/lld/test/MachO/set-slop-scale.s
deleted file mode 100644
index a18acce4b4543..0000000000000
--- a/lld/test/MachO/set-slop-scale.s
+++ /dev/null
@@ -1,11 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-darwin %s -o %t.o
-# RUN: %lld -o /dev/null %t.o --slop_scale=1
-# RUN: not %lld -o /dev/null %t.o --slop_scale=-1 2>&1 | FileCheck %s
-# CHECK: error: --slop_scale=: expected a non-negative integer, but got '-1'
-
-.text
-.global _main
-_main:
- mov $0, %rax
- ret
>From 66119a3376410f43a6465054f9dc670ceb99249c Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 21 Apr 2026 17:59:30 -0700
Subject: [PATCH 06/17] remove dead code
---
lld/MachO/ConcatOutputSection.cpp | 6 ------
1 file changed, 6 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index d9ad94b57acc2..b84d4baacf290 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -283,12 +283,6 @@ void TextOutputSection::finalize() {
}
return isTargetInRange(*callerIsec, *r);
});
- // Count the number of new thunks we will need to create
- DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
- for (auto [callerIsec, r, thunkKey] : branchesToProcess)
- if (!getThunkInRange(*callerIsec, *r))
- branchTargets.insert(thunkKey);
- assert(numPendingThunkTargets == branchTargets.size());
uint64_t estimatedTextEnd = addr + size + numPendingThunkTargets * thunkSize;
uint64_t estimatedStubsEnd =
>From 25f4038741bf0eab80bc3a717d76ae64a5f52c4c Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Fri, 24 Apr 2026 13:36:46 -0700
Subject: [PATCH 07/17] check stub targets in deferredBranchRedirects
---
lld/MachO/ConcatOutputSection.cpp | 67 +++++++++++++++++++------------
lld/MachO/InputSection.h | 1 -
2 files changed, 42 insertions(+), 26 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index b312bf403152d..63e0c1223264f 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -119,8 +119,8 @@ void TextOutputSection::finalize() {
size_t thunkCallCount = 0;
size_t thunkCount = 0;
- auto isTargetInRange = [&](const ConcatInputSection &isec,
- const Relocation &r) -> bool {
+ auto isTargetKnownInRange = [&](const ConcatInputSection &isec,
+ const Relocation &r) -> bool {
uint64_t callVA = isec.getVA() + r.offset;
uint64_t lowVA =
backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
@@ -132,7 +132,7 @@ void TextOutputSection::finalize() {
};
auto getThunkInRange = [&](const ConcatInputSection &isec,
const Relocation &r) -> Defined * {
- assert(!isTargetInRange(isec, r));
+ assert(!isTargetKnownInRange(isec, r));
uint64_t callVA = isec.getVA() + r.offset;
uint64_t lowVA =
backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
@@ -153,7 +153,7 @@ void TextOutputSection::finalize() {
if (addr + size > highVA) {
// There were too many consecutive branch instructions for `slop`
// below. If you hit this: For the current algorithm, just bumping up
- // slop above and trying again is probably simplest. (See also PR51578
+ // slop below and trying again is probably simplest. (See also PR51578
// comment 5).
fatal(Twine(__FUNCTION__) +
": FIXME: thunk range overrun. Consider increasing the "
@@ -196,8 +196,13 @@ void TextOutputSection::finalize() {
++thunkCount;
};
+ // Branches whose target sections are out of range or have not yet been
+ // finalized. We may need to emit thunks for them.
std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
branchesToProcess;
+ // Branches whose targets have not yet be finalized, but a thunk for that
+ // target exists. We defer processing these branches because it's possible we
+ // can still direct call to their targets after they have all been finalized.
SmallVector<std::tuple<ConcatInputSection *, Relocation *, Defined *>>
deferredBranchRedirects;
@@ -206,7 +211,7 @@ void TextOutputSection::finalize() {
while (!branchesToProcess.empty()) {
auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
assert(callerIsec->isFinal);
- if (isTargetInRange(*callerIsec, *r)) {
+ if (isTargetKnownInRange(*callerIsec, *r)) {
branchesToProcess.pop_front();
continue;
}
@@ -218,6 +223,10 @@ void TextOutputSection::finalize() {
uint64_t highVA = callerIsec->getVA() + r->offset + forwardBranchRange;
uint64_t nextEnd =
alignToPowerOf2(addr + size, isec->align) + isec->getSize();
+ // If we were to emit this section, would we have enough space for more
+ // thunks? If we do, then we can delay processing this thunk so we may
+ // finalize more potencial target sections. Otherwise we must emit thunks
+ // until we have enough space.
if (nextEnd + slop <= highVA)
break;
@@ -241,7 +250,7 @@ void TextOutputSection::finalize() {
for (Relocation &r : reverse(isec->relocs)) {
if (!target->hasAttr(r.type, RelocAttrBits::BRANCH))
continue;
- if (isTargetInRange(*isec, r))
+ if (isTargetKnownInRange(*isec, r))
continue;
if (auto *sym = getThunkInRange(*isec, r)) {
deferredBranchRedirects.emplace_back(isec, &r, sym);
@@ -252,20 +261,14 @@ void TextOutputSection::finalize() {
branchesToProcess.emplace_back(isec, &r, key);
}
}
- for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
- if (isTargetInRange(*callerIsec, *r))
- continue;
- r->referent = thunk;
- r->addend = 0;
- ++thunkCallCount;
- }
- deferredBranchRedirects.clear();
llvm::erase_if(branchesToProcess, [&](auto &tuple) {
auto [callerIsec, r, thunkKey] = tuple;
- return isTargetInRange(*callerIsec, *r);
+ return isTargetKnownInRange(*callerIsec, *r);
});
- // Count the number of new thunks we will need to create
+ // Count distinct unresolved branch targets that still lack an in-range thunk.
+ // We use this as an upper bound on the number of thunks we may still create
+ // when estimating where __stubs / __objc_stubs could end up.
DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
for (auto [callerIsec, r, thunkKey] : branchesToProcess)
if (!getThunkInRange(*callerIsec, *r))
@@ -279,17 +282,31 @@ void TextOutputSection::finalize() {
alignToPowerOf2(estimatedStubsEnd, in.objcStubs->align) +
in.objcStubs->getSize();
+ auto isTargetStubsAndInRange = [&](const ConcatInputSection &isec,
+ const Relocation &r) -> bool {
+ auto *funcSym = cast<Symbol *>(r.referent);
+ if (!funcSym->isInStubs() && !(in.objcStubs && in.objcStubs->isNeeded() &&
+ ObjCStubsSection::isObjCStubSymbol(funcSym)))
+ return false;
+ if (r.addend)
+ return false;
+ uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
+ return estimatedStubsEnd <= highVA;
+ };
+
+ for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
+ if (isTargetKnownInRange(*callerIsec, *r))
+ continue;
+ if (isTargetStubsAndInRange(*callerIsec, *r))
+ continue;
+ r->referent = thunk;
+ r->addend = 0;
+ ++thunkCallCount;
+ }
+
for (auto [isec, r, thunkKey] : branchesToProcess) {
- uint64_t highVA = isec->getVA() + r->offset + forwardBranchRange;
- auto *funcSym = cast<Symbol *>(r->referent);
- if ((funcSym->isInStubs() ||
- (in.objcStubs && in.objcStubs->isNeeded() &&
- ObjCStubsSection::isObjCStubSymbol(funcSym))) &&
- r->addend == 0 && estimatedStubsEnd <= highVA) {
- // The branch target can reach any section in __stubs or __objc_stubs
- assert(isec->getVA() != TargetInfo::outOfRangeVA);
+ if (isTargetStubsAndInRange(*isec, *r))
continue;
- }
if (auto *sym = getThunkInRange(*isec, *r)) {
r->referent = sym;
// The thunk itself bakes in the addend, so the call-site reloc must
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 2ecc198c99c5d..89500257a3fd3 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -142,7 +142,6 @@ class ConcatInputSection final : public InputSection {
// first and not copied to the output.
bool wasCoalesced = false;
bool live = !config->deadStrip;
- bool hasCallSites = false;
// This variable has two usages. Initially, it represents the input order.
// After assignAddresses is called, it represents the offset from the
// beginning of the output section this section was assigned to.
>From 3c5b94f69d985587a4bfbd0360c2d135894df8ed Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Fri, 24 Apr 2026 14:24:51 -0700
Subject: [PATCH 08/17] add updateBranchTargetToThunk
---
lld/MachO/ConcatOutputSection.cpp | 46 +++++++++++++++----------------
1 file changed, 22 insertions(+), 24 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 63e0c1223264f..8de59cfc56744 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -117,7 +117,6 @@ void TextOutputSection::finalize() {
uint64_t backwardBranchRange = target->backwardBranchRange;
size_t thunkSize = target->thunkSize;
size_t thunkCallCount = 0;
- size_t thunkCount = 0;
auto isTargetKnownInRange = [&](const ConcatInputSection &isec,
const Relocation &r) -> bool {
@@ -146,6 +145,13 @@ void TextOutputSection::finalize() {
}
return nullptr;
};
+ auto updateBranchTargetToThunk = [&](Relocation &r, Defined *thunk) {
+ r.referent = thunk;
+ // The thunk itself bakes in the addend, so the call-site reloc must
+ // branch to the thunk start with no extra offset.
+ r.addend = 0;
+ ++thunkCallCount;
+ };
auto createThunk = [&](const ConcatInputSection &isec, Relocation &r) {
assert(getThunkInRange(isec, r) == nullptr);
assert(isec.isFinal);
@@ -173,13 +179,13 @@ void TextOutputSection::finalize() {
saver().save(funcSym->getName() + addendSuffix + ".thunk." +
std::to_string(thunkInfo.sequence++));
if (!isa<Defined>(funcSym) || cast<Defined>(funcSym)->isExternal()) {
- r.referent = thunkInfo.sym = symtab->addDefined(
+ thunkInfo.sym = symtab->addDefined(
thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
/*isWeakDef=*/false, /*isPrivateExtern=*/true,
/*isReferencedDynamically=*/false, /*noDeadStrip=*/false,
/*isWeakDefCanBeHidden=*/false);
} else {
- r.referent = thunkInfo.sym = make<Defined>(
+ thunkInfo.sym = make<Defined>(
thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
/*isWeakDef=*/false, /*isExternal=*/false, /*isPrivateExtern=*/true,
/*includeInSymtab=*/true, /*isReferencedDynamically=*/false,
@@ -187,13 +193,9 @@ void TextOutputSection::finalize() {
}
thunkInfo.sym->used = true;
target->populateThunk(thunkInfo.isec, funcSym, r.addend);
- // The thunk itself bakes in the addend, so the call-site reloc must
- // branch to the thunk start with no extra offset.
- r.addend = 0;
- ++thunkCallCount;
+ updateBranchTargetToThunk(r, thunkInfo.sym);
finalizeOne(thunkInfo.isec);
thunks.push_back(thunkInfo.isec);
- ++thunkCount;
};
// Branches whose target sections are out of range or have not yet been
@@ -215,8 +217,8 @@ void TextOutputSection::finalize() {
branchesToProcess.pop_front();
continue;
}
- if (auto *sym = getThunkInRange(*callerIsec, *r)) {
- deferredBranchRedirects.emplace_back(callerIsec, r, sym);
+ if (auto *thunk = getThunkInRange(*callerIsec, *r)) {
+ deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
branchesToProcess.pop_front();
continue;
}
@@ -235,6 +237,8 @@ void TextOutputSection::finalize() {
}
finalizeOne(isec);
+ // TODO: Remove this check and the assert below. In fact, I don't believe
+ // the relocation iteration order matters for correctness.
bool hasCallsite = llvm::any_of(isec->relocs, [](Relocation &r) {
return target->hasAttr(r.type, RelocAttrBits::BRANCH);
});
@@ -252,8 +256,8 @@ void TextOutputSection::finalize() {
continue;
if (isTargetKnownInRange(*isec, r))
continue;
- if (auto *sym = getThunkInRange(*isec, r)) {
- deferredBranchRedirects.emplace_back(isec, &r, sym);
+ if (auto *thunk = getThunkInRange(*isec, r)) {
+ deferredBranchRedirects.emplace_back(isec, &r, thunk);
continue;
}
auto *funcSym = cast<Symbol *>(r.referent);
@@ -299,28 +303,22 @@ void TextOutputSection::finalize() {
continue;
if (isTargetStubsAndInRange(*callerIsec, *r))
continue;
- r->referent = thunk;
- r->addend = 0;
- ++thunkCallCount;
+ updateBranchTargetToThunk(*r, thunk);
}
for (auto [isec, r, thunkKey] : branchesToProcess) {
if (isTargetStubsAndInRange(*isec, *r))
continue;
- if (auto *sym = getThunkInRange(*isec, *r)) {
- r->referent = sym;
- // The thunk itself bakes in the addend, so the call-site reloc must
- // branch to the thunk start with no extra offset.
- r->addend = 0;
- ++thunkCallCount;
+ if (auto *thunk = getThunkInRange(*isec, *r)) {
+ updateBranchTargetToThunk(*r, thunk);
continue;
}
createThunk(*isec, *r);
}
- if (thunkCount)
- log("Created " + Twine(thunkCount) + " (" +
- Twine(thunkCount * thunkSize / 1024) + " KB) thunks and updated " +
+ if (thunks.size())
+ log("Created " + Twine(thunks.size()) + " (" +
+ Twine(thunks.size() * thunkSize / 1024) + " KB) thunks and updated " +
Twine(thunkCallCount) + " branch targets");
}
>From 18fc429b5d04c7212d5a8d03217fadd27b07ee1a Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Fri, 24 Apr 2026 15:58:48 -0700
Subject: [PATCH 09/17] add missing test checks
---
lld/test/MachO/arm64-thunks.s | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/lld/test/MachO/arm64-thunks.s b/lld/test/MachO/arm64-thunks.s
index f482f01a690d4..9096aaef78472 100644
--- a/lld/test/MachO/arm64-thunks.s
+++ b/lld/test/MachO/arm64-thunks.s
@@ -69,6 +69,8 @@
# CHECK: Disassembly of section __TEXT,__text:
+# CHECK: [[#%.13x, FOLD_LOW_PAGE:]][[#%.3x, FOLD_LOW_OFFSET:]] <_fold_func_low_addr>:
+
# CHECK: [[#%.13x, A_PAGE:]][[#%.3x, A_OFFSET:]] <_a>:
# CHECK: bl 0x[[#%x, A:]] <_a>
# CHECK: bl 0x[[#%x, B:]] <_b>
@@ -168,7 +170,7 @@
# CHECK: bl 0x[[#%x, F]] <_f>
# CHECK: bl 0x[[#%x, G]] <_g>
# CHECK: bl 0x[[#%x, H]] <_h>
-# CHECK: bl 0x[[#%x, STUBS:]]
+# CHECK: bl 0x[[#%x, NAN:]]
# CHECK: [[#%x, A_THUNK_0]] <_a.thunk.0>:
# CHECK: adrp x16, 0x[[#%x, A_PAGE]]000
@@ -187,7 +189,7 @@
# CHECK: bl 0x[[#%x, F]] <_f>
# CHECK: bl 0x[[#%x, G]] <_g>
# CHECK: bl 0x[[#%x, H]] <_h>
-# CHECK: bl 0x[[#%x, STUBS]]
+# CHECK: bl 0x[[#%x, NAN]]
# CHECK: <_main>:
# CHECK: bl 0x[[#%x, A_THUNK_0]] <_a.thunk.0>
@@ -198,7 +200,14 @@
# CHECK: bl 0x[[#%x, F_THUNK_1:]] <_f.thunk.1>
# CHECK: bl 0x[[#%x, G]] <_g>
# CHECK: bl 0x[[#%x, H]] <_h>
-# CHECK: bl 0x[[#%x, STUBS]]
+# CHECK: bl 0x[[#%x, FOLD_LOW_THUNK_0:]] <_fold_func_low_addr.thunk.0>
+# CHECK: bl 0x[[#%x, FOLD_HIGH:]] <_fold_func_high_addr>
+# CHECK: bl 0x[[#%x, NAN]]
+# CHECK: bl 0x[[#%x, FOO:]] <_objc_msgSend$foo>
+# CHECK: bl 0x[[#%x, BAR:]] <_objc_msgSend$bar>
+
+# CHECK: [[#%x, FOLD_HIGH]] <_fold_func_high_addr>:
+# CHECK: b 0x[[#%x, FOLD_LOW_THUNK_0]] <_fold_func_low_addr.thunk.0>
# CHECK: [[#%x, C_THUNK_0]] <_c.thunk.0>:
# CHECK: adrp x16, 0x[[#%x, C_PAGE]]000
@@ -216,6 +225,10 @@
# CHECK: adrp x16, 0x[[#%x, F_PAGE]]
# CHECK: add x16, x16, #[[#F_OFFSET]]
+# CHECK: [[#%x, FOLD_LOW_THUNK_0]] <_fold_func_low_addr.thunk.0>:
+# CHECK: adrp x16, 0x[[#%x, FOLD_LOW_PAGE]]000
+# CHECK: add x16, x16, #[[#%d, FOLD_LOW_OFFSET]]
+
# CHECK: Disassembly of section __TEXT,__lcxx_override:
# CHECK: <_z>:
# CHECK: bl 0x[[#%x, A_THUNK_0]] <_a.thunk.0>
@@ -224,6 +237,11 @@
# CHECK: [[#%x, NAN_PAGE + NAN_OFFSET]] <__stubs>:
+# CHECK: Disassembly of section __TEXT,__objc_stubs:
+
+# CHECK: <_objc_msgSend$bar>:
+# CHECK: <_objc_msgSend$foo>:
+
.section __TEXT,__objc_methname,cstring_literals
lselref1:
.asciz "foo"
>From ae118427fee465d4ca856bc43425996ef75af394 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Mon, 27 Apr 2026 14:35:18 -0700
Subject: [PATCH 10/17] move lambdas to private helpers
---
lld/MachO/ConcatOutputSection.cpp | 217 +++++++++++++++---------------
lld/MachO/ConcatOutputSection.h | 22 ++-
2 files changed, 132 insertions(+), 107 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 8de59cfc56744..6f7c2e89a81fa 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -106,6 +106,108 @@ void ConcatOutputSection::finalizeContents() {
finalizeOne(isec);
}
+bool TextOutputSection::isTargetKnownInRange(const ConcatInputSection &isec,
+ const Relocation &r) const {
+ uint64_t callVA = isec.getVA() + r.offset;
+ uint64_t lowVA = target->backwardBranchRange < callVA
+ ? callVA - target->backwardBranchRange
+ : 0;
+ uint64_t highVA = callVA + target->forwardBranchRange;
+ auto *funcSym = cast<Symbol *>(r.referent);
+ uint64_t funcVA = resolveSymbolOffsetVA(funcSym, r.type, r.addend);
+ // Check if the referent is reachable with a simple call instruction.
+ return lowVA <= funcVA && funcVA <= highVA;
+}
+
+Defined *TextOutputSection::getThunkInRange(const ConcatInputSection &isec,
+ const Relocation &r) const {
+ assert(!isTargetKnownInRange(isec, r));
+ uint64_t callVA = isec.getVA() + r.offset;
+ uint64_t lowVA = target->backwardBranchRange < callVA
+ ? callVA - target->backwardBranchRange
+ : 0;
+ uint64_t highVA = callVA + target->forwardBranchRange;
+ auto *funcSym = cast<Symbol *>(r.referent);
+ auto it = thunkMap.find(ThunkKey{funcSym, r.addend});
+ if (it == thunkMap.end())
+ return nullptr;
+ auto &thunkInfo = it->second;
+ assert(thunkInfo.sym);
+ uint64_t thunkVA = thunkInfo.isec->getVA();
+ if (lowVA <= thunkVA && thunkVA <= highVA)
+ return thunkInfo.sym;
+ return nullptr;
+}
+
+void TextOutputSection::updateBranchTargetToThunk(Relocation &r,
+ Defined *thunk) {
+ r.referent = thunk;
+ // The thunk itself bakes in the addend, so the call-site reloc must
+ // branch to the thunk start with no extra offset.
+ r.addend = 0;
+ ++thunkCallCount;
+}
+
+void TextOutputSection::createThunk(const ConcatInputSection &isec,
+ Relocation &r) {
+ assert(getThunkInRange(isec, r) == nullptr);
+ assert(isec.isFinal);
+ uint64_t highVA = isec.getVA() + r.offset + target->forwardBranchRange;
+ if (addr + size > highVA) {
+ // There were too many consecutive branch instructions for `slop`
+ // below. If you hit this: For the current algorithm, just bumping up
+ // slop below and trying again is probably simplest. (See also PR51578
+ // comment 5).
+ fatal(Twine(__FUNCTION__) +
+ ": FIXME: thunk range overrun. Consider increasing the "
+ "slop-scale with `--slop-scale=<unsigned_int>`.");
+ }
+ auto *funcSym = cast<Symbol *>(r.referent);
+ ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
+ thunkInfo.isec = makeSyntheticInputSection(isec.getSegName(), isec.getName());
+ thunkInfo.isec->parent = this;
+ assert(thunkInfo.isec->live);
+
+ std::string addendSuffix;
+ if (r.addend != 0)
+ addendSuffix = "+" + std::to_string(r.addend);
+ size_t thunkSize = target->thunkSize;
+ StringRef thunkName =
+ saver().save(funcSym->getName() + addendSuffix + ".thunk." +
+ std::to_string(thunkInfo.sequence++));
+ if (!isa<Defined>(funcSym) || cast<Defined>(funcSym)->isExternal()) {
+ thunkInfo.sym = symtab->addDefined(
+ thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
+ /*isWeakDef=*/false, /*isPrivateExtern=*/true,
+ /*isReferencedDynamically=*/false, /*noDeadStrip=*/false,
+ /*isWeakDefCanBeHidden=*/false);
+ } else {
+ thunkInfo.sym = make<Defined>(
+ thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
+ /*isWeakDef=*/false, /*isExternal=*/false, /*isPrivateExtern=*/true,
+ /*includeInSymtab=*/true, /*isReferencedDynamically=*/false,
+ /*noDeadStrip=*/false, /*isWeakDefCanBeHidden=*/false);
+ }
+ thunkInfo.sym->used = true;
+ target->populateThunk(thunkInfo.isec, funcSym, r.addend);
+ updateBranchTargetToThunk(r, thunkInfo.sym);
+ finalizeOne(thunkInfo.isec);
+ thunks.push_back(thunkInfo.isec);
+}
+
+bool TextOutputSection::isTargetStubsAndInRange(
+ const ConcatInputSection &isec, const Relocation &r,
+ uint64_t estimatedStubsEnd) const {
+ auto *funcSym = cast<Symbol *>(r.referent);
+ if (!funcSym->isInStubs() && !(in.objcStubs && in.objcStubs->isNeeded() &&
+ ObjCStubsSection::isObjCStubSymbol(funcSym)))
+ return false;
+ if (r.addend)
+ return false;
+ uint64_t highVA = isec.getVA() + r.offset + target->forwardBranchRange;
+ return estimatedStubsEnd <= highVA;
+}
+
void TextOutputSection::finalize() {
if (!needsThunks()) {
for (ConcatInputSection *isec : inputs)
@@ -113,91 +215,6 @@ void TextOutputSection::finalize() {
return;
}
- uint64_t forwardBranchRange = target->forwardBranchRange;
- uint64_t backwardBranchRange = target->backwardBranchRange;
- size_t thunkSize = target->thunkSize;
- size_t thunkCallCount = 0;
-
- auto isTargetKnownInRange = [&](const ConcatInputSection &isec,
- const Relocation &r) -> bool {
- uint64_t callVA = isec.getVA() + r.offset;
- uint64_t lowVA =
- backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
- uint64_t highVA = callVA + forwardBranchRange;
- auto *funcSym = cast<Symbol *>(r.referent);
- uint64_t funcVA = resolveSymbolOffsetVA(funcSym, r.type, r.addend);
- // Check if the referent is reachable with a simple call instruction.
- return lowVA <= funcVA && funcVA <= highVA;
- };
- auto getThunkInRange = [&](const ConcatInputSection &isec,
- const Relocation &r) -> Defined * {
- assert(!isTargetKnownInRange(isec, r));
- uint64_t callVA = isec.getVA() + r.offset;
- uint64_t lowVA =
- backwardBranchRange < callVA ? callVA - backwardBranchRange : 0;
- uint64_t highVA = callVA + forwardBranchRange;
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
- if (thunkInfo.sym) {
- uint64_t thunkVA = thunkInfo.isec->getVA();
- if (lowVA <= thunkVA && thunkVA <= highVA)
- return thunkInfo.sym;
- }
- return nullptr;
- };
- auto updateBranchTargetToThunk = [&](Relocation &r, Defined *thunk) {
- r.referent = thunk;
- // The thunk itself bakes in the addend, so the call-site reloc must
- // branch to the thunk start with no extra offset.
- r.addend = 0;
- ++thunkCallCount;
- };
- auto createThunk = [&](const ConcatInputSection &isec, Relocation &r) {
- assert(getThunkInRange(isec, r) == nullptr);
- assert(isec.isFinal);
- uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
- if (addr + size > highVA) {
- // There were too many consecutive branch instructions for `slop`
- // below. If you hit this: For the current algorithm, just bumping up
- // slop below and trying again is probably simplest. (See also PR51578
- // comment 5).
- fatal(Twine(__FUNCTION__) +
- ": FIXME: thunk range overrun. Consider increasing the "
- "slop-scale with `--slop-scale=<unsigned_int>`.");
- }
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
- thunkInfo.isec =
- makeSyntheticInputSection(isec.getSegName(), isec.getName());
- thunkInfo.isec->parent = this;
- assert(thunkInfo.isec->live);
-
- std::string addendSuffix;
- if (r.addend != 0)
- addendSuffix = "+" + std::to_string(r.addend);
- StringRef thunkName =
- saver().save(funcSym->getName() + addendSuffix + ".thunk." +
- std::to_string(thunkInfo.sequence++));
- if (!isa<Defined>(funcSym) || cast<Defined>(funcSym)->isExternal()) {
- thunkInfo.sym = symtab->addDefined(
- thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
- /*isWeakDef=*/false, /*isPrivateExtern=*/true,
- /*isReferencedDynamically=*/false, /*noDeadStrip=*/false,
- /*isWeakDefCanBeHidden=*/false);
- } else {
- thunkInfo.sym = make<Defined>(
- thunkName, /*file=*/nullptr, thunkInfo.isec, /*value=*/0, thunkSize,
- /*isWeakDef=*/false, /*isExternal=*/false, /*isPrivateExtern=*/true,
- /*includeInSymtab=*/true, /*isReferencedDynamically=*/false,
- /*noDeadStrip=*/false, /*isWeakDefCanBeHidden=*/false);
- }
- thunkInfo.sym->used = true;
- target->populateThunk(thunkInfo.isec, funcSym, r.addend);
- updateBranchTargetToThunk(r, thunkInfo.sym);
- finalizeOne(thunkInfo.isec);
- thunks.push_back(thunkInfo.isec);
- };
-
// Branches whose target sections are out of range or have not yet been
// finalized. We may need to emit thunks for them.
std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
@@ -208,7 +225,7 @@ void TextOutputSection::finalize() {
SmallVector<std::tuple<ConcatInputSection *, Relocation *, Defined *>>
deferredBranchRedirects;
- const uint64_t slop = config->slopScale * thunkSize;
+ const uint64_t slop = config->slopScale * target->thunkSize;
for (auto *isec : inputs) {
while (!branchesToProcess.empty()) {
auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
@@ -222,7 +239,8 @@ void TextOutputSection::finalize() {
branchesToProcess.pop_front();
continue;
}
- uint64_t highVA = callerIsec->getVA() + r->offset + forwardBranchRange;
+ uint64_t highVA =
+ callerIsec->getVA() + r->offset + target->forwardBranchRange;
uint64_t nextEnd =
alignToPowerOf2(addr + size, isec->align) + isec->getSize();
// If we were to emit this section, would we have enough space for more
@@ -278,7 +296,8 @@ void TextOutputSection::finalize() {
if (!getThunkInRange(*callerIsec, *r))
branchTargets.insert(thunkKey);
- uint64_t estimatedTextEnd = addr + size + branchTargets.size() * thunkSize;
+ uint64_t estimatedTextEnd =
+ addr + size + branchTargets.size() * target->thunkSize;
uint64_t estimatedStubsEnd =
alignToPowerOf2(estimatedTextEnd, in.stubs->align) + in.stubs->getSize();
if (in.objcStubs && in.objcStubs->isNeeded())
@@ -286,28 +305,16 @@ void TextOutputSection::finalize() {
alignToPowerOf2(estimatedStubsEnd, in.objcStubs->align) +
in.objcStubs->getSize();
- auto isTargetStubsAndInRange = [&](const ConcatInputSection &isec,
- const Relocation &r) -> bool {
- auto *funcSym = cast<Symbol *>(r.referent);
- if (!funcSym->isInStubs() && !(in.objcStubs && in.objcStubs->isNeeded() &&
- ObjCStubsSection::isObjCStubSymbol(funcSym)))
- return false;
- if (r.addend)
- return false;
- uint64_t highVA = isec.getVA() + r.offset + forwardBranchRange;
- return estimatedStubsEnd <= highVA;
- };
-
for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
if (isTargetKnownInRange(*callerIsec, *r))
continue;
- if (isTargetStubsAndInRange(*callerIsec, *r))
+ if (isTargetStubsAndInRange(*callerIsec, *r, estimatedStubsEnd))
continue;
updateBranchTargetToThunk(*r, thunk);
}
for (auto [isec, r, thunkKey] : branchesToProcess) {
- if (isTargetStubsAndInRange(*isec, *r))
+ if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
if (auto *thunk = getThunkInRange(*isec, *r)) {
updateBranchTargetToThunk(*r, thunk);
@@ -316,10 +323,10 @@ void TextOutputSection::finalize() {
createThunk(*isec, *r);
}
- if (thunks.size())
+ if (!thunks.empty())
log("Created " + Twine(thunks.size()) + " (" +
- Twine(thunks.size() * thunkSize / 1024) + " KB) thunks and updated " +
- Twine(thunkCallCount) + " branch targets");
+ Twine(thunks.size() * target->thunkSize / 1024) +
+ " KB) thunks and updated " + Twine(thunkCallCount) + " branch targets");
}
void ConcatOutputSection::writeTo(uint8_t *buf) const {
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index 154272392928a..5693b1416e061 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -79,9 +79,27 @@ class TextOutputSection : public ConcatOutputSection {
}
private:
- uint64_t estimateBranchTargetThresholdVA(size_t callIdx) const;
-
std::vector<ConcatInputSection *> thunks;
+ /// \return true if the target in \p r is in range from the location in \p
+ /// isec. If the target isec is not finalized, \return false.
+ bool isTargetKnownInRange(const ConcatInputSection &isec,
+ const Relocation &r) const;
+ /// If there exists a thunk in range of the target in \p r, \return that
+ /// thunk.
+ Defined *getThunkInRange(const ConcatInputSection &isec,
+ const Relocation &r) const;
+ /// Update \p r to target \p thunk which is guaranteed to be in range.
+ void updateBranchTargetToThunk(Relocation &r, Defined *thunk);
+ /// Create a new thunk and update \p r to target the new thunk.
+ void createThunk(const ConcatInputSection &isec, Relocation &r);
+ /// \return true if the target in \p r is in __stubs or __objc_stubs and in
+ /// range from the location in \p isec. \p estimatedStubsEnd is the estimated
+ /// VA of the end of the last stubs section.
+ bool isTargetStubsAndInRange(const ConcatInputSection &isec,
+ const Relocation &r,
+ uint64_t estimatedStubsEnd) const;
+ /// The number of relocations updated to point to thunks.
+ size_t thunkCallCount = 0;
};
// We maintain one ThunkInfo per real function.
>From c76374a671da468d9dea7cf505ad7a0bfd279f93 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 28 Apr 2026 09:50:31 -0700
Subject: [PATCH 11/17] add section name to log
---
lld/MachO/ConcatOutputSection.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 6f7c2e89a81fa..f62eebc744551 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -324,7 +324,7 @@ void TextOutputSection::finalize() {
}
if (!thunks.empty())
- log("Created " + Twine(thunks.size()) + " (" +
+ log(name + ": Created " + Twine(thunks.size()) + " (" +
Twine(thunks.size() * target->thunkSize / 1024) +
" KB) thunks and updated " + Twine(thunkCallCount) + " branch targets");
}
>From 88cb362f304d6eb0410f8dee3ccdaa8431ff5f09 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Wed, 29 Apr 2026 15:10:06 -0700
Subject: [PATCH 12/17] resolve some comments
---
lld/MachO/ConcatOutputSection.cpp | 19 +++++++++----------
lld/MachO/Driver.cpp | 2 ++
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index b5b664e07d1bc..8bceeb90b0ff1 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -152,11 +152,11 @@ void TextOutputSection::createThunk(const ConcatInputSection &isec,
assert(isec.isFinal);
uint64_t highVA = isec.getVA() + r.offset + target->forwardBranchRange;
if (addr + size > highVA) {
- // We can only encounter this if we have a massive section (> ~128MB) or
- // an enormous number of branch instructions within a single section
- // (> ~16M), neither of which is feasible in practice. To fix we could
- // implement branch islands when the available space for thunks become too
- // small.
+ // We can only encounter this if we have a massive function
+ // (> ~128MB on arm64) or an enormous number of branch instructions within a
+ // single function (> ~16M on arm64), neither of which is feasible in
+ // practice. To fix we could implement branch islands when the available
+ // space for thunks become too small.
fatal("encountered a branch whose target is out of range, but there is "
"no more space for a new thunk");
}
@@ -250,7 +250,9 @@ void TextOutputSection::finalize() {
if (nextEnd + numPendingThunkTargets * target->thunkSize <= highVA)
break;
+ assert(thunkInfo.pendingBranches.size());
thunkInfo.pendingBranches.clear();
+ assert(numPendingThunkTargets);
--numPendingThunkTargets;
createThunk(*callerIsec, *r);
branchesToProcess.pop_front();
@@ -299,6 +301,8 @@ void TextOutputSection::finalize() {
if (thunkInfo.pendingBranches.empty())
--numPendingThunkTargets;
}
+ // Do not remove branches if a thunk is in range because if the target is a
+ // stub we may discover that it is in range for a direct branch
return targetInRange;
});
@@ -328,10 +332,6 @@ void TextOutputSection::finalize() {
}
for (auto [isec, r, thunkKey] : branchesToProcess) {
- auto &thunkInfo = thunkMap[thunkKey];
- if (thunkInfo.pendingBranches.erase(r))
- if (thunkInfo.pendingBranches.empty())
- --numPendingThunkTargets;
if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
if (auto *thunk = getThunkInRange(*isec, *r)) {
@@ -340,7 +340,6 @@ void TextOutputSection::finalize() {
}
createThunk(*isec, *r);
}
- assert(numPendingThunkTargets == 0);
if (!thunks.empty())
log(name + ": Created " + Twine(thunks.size()) + " (" +
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 9d8cbb43b64ea..adce6b6dc6717 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -2015,6 +2015,8 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
OPT_no_separate_cstring_literal_sections, false);
config->tailMergeStrings =
args.hasFlag(OPT_tail_merge_strings, OPT_no_tail_merge_strings, false);
+ if (auto *arg = args.getLastArg(OPT_slop_scale_eq))
+ warn(arg->getSpelling() + " has no effect and is deprecated");
auto IncompatWithCGSort = [&](StringRef firstArgStr) {
// Throw an error only if --call-graph-profile-sort is explicitly specified
>From 49d1797caa44e454695347989475459428f89578 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Wed, 29 Apr 2026 16:02:01 -0700
Subject: [PATCH 13/17] Add helper function
---
lld/MachO/ConcatOutputSection.cpp | 41 ++++++++++++++-----------
lld/MachO/ConcatOutputSection.h | 50 ++++++++++++++++++-------------
2 files changed, 53 insertions(+), 38 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 8bceeb90b0ff1..3bf92807bfb09 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -206,6 +206,16 @@ bool TextOutputSection::isTargetStubsAndInRange(
return estimatedStubsEnd <= highVA;
}
+void TextOutputSection::markBranchAsResolved(ThunkInfo &thunkInfo,
+ const Relocation *r) {
+ if (!thunkInfo.pendingBranches.erase(r))
+ return;
+ if (!thunkInfo.pendingBranches.empty())
+ return;
+ assert(numPendingThunkTargets);
+ --numPendingThunkTargets;
+}
+
void TextOutputSection::finalize() {
if (!needsThunks()) {
for (ConcatInputSection *isec : inputs)
@@ -222,7 +232,6 @@ void TextOutputSection::finalize() {
// can still direct call to their targets after they have all been finalized.
SmallVector<std::tuple<ConcatInputSection *, Relocation *, Defined *>>
deferredBranchRedirects;
- unsigned numPendingThunkTargets = 0;
for (auto *isec : inputs) {
while (!branchesToProcess.empty()) {
@@ -231,14 +240,10 @@ void TextOutputSection::finalize() {
auto &thunkInfo = thunkMap[thunkKey];
if (isTargetKnownInRange(*callerIsec, *r)) {
branchesToProcess.pop_front();
- if (thunkInfo.pendingBranches.erase(r))
- if (thunkInfo.pendingBranches.empty())
- --numPendingThunkTargets;
+ markBranchAsResolved(thunkInfo, r);
continue;
}
if (auto *thunk = getThunkInRange(*callerIsec, *r)) {
- // The pending thunk target was already decremented when we created the
- // thunk
deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
branchesToProcess.pop_front();
continue;
@@ -247,10 +252,14 @@ void TextOutputSection::finalize() {
callerIsec->getVA() + r->offset + target->forwardBranchRange;
uint64_t nextEnd =
alignToPowerOf2(addr + size, isec->align) + isec->getSize();
+ // If we were to emit this section, would we have enough space for more
+ // thunks? If we do, then we can delay processing this thunk so we may
+ // finalize more potencial target sections. Otherwise we must emit thunks
+ // until we have enough space.
if (nextEnd + numPendingThunkTargets * target->thunkSize <= highVA)
break;
- assert(thunkInfo.pendingBranches.size());
+ assert(thunkInfo.pendingBranches.contains(r));
thunkInfo.pendingBranches.clear();
assert(numPendingThunkTargets);
--numPendingThunkTargets;
@@ -294,16 +303,10 @@ void TextOutputSection::finalize() {
llvm::erase_if(branchesToProcess, [&](auto &tuple) {
auto [callerIsec, r, thunkKey] = tuple;
- bool targetInRange = isTargetKnownInRange(*callerIsec, *r);
- auto &thunkInfo = thunkMap[thunkKey];
- if (targetInRange || getThunkInRange(*callerIsec, *r)) {
- if (thunkInfo.pendingBranches.erase(r))
- if (thunkInfo.pendingBranches.empty())
- --numPendingThunkTargets;
- }
- // Do not remove branches if a thunk is in range because if the target is a
- // stub we may discover that it is in range for a direct branch
- return targetInRange;
+ if (!isTargetKnownInRange(*callerIsec, *r))
+ return false;
+ markBranchAsResolved(thunkMap[thunkKey], r);
+ return true;
});
#ifndef NDEBUG
@@ -332,6 +335,9 @@ void TextOutputSection::finalize() {
}
for (auto [isec, r, thunkKey] : branchesToProcess) {
+#ifndef NDEBUG
+ markBranchAsResolved(thunkMap[thunkKey], r);
+#endif
if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
if (auto *thunk = getThunkInRange(*isec, *r)) {
@@ -340,6 +346,7 @@ void TextOutputSection::finalize() {
}
createThunk(*isec, *r);
}
+ assert(numPendingThunkTargets == 0);
if (!thunks.empty())
log(name + ": Created " + Twine(thunks.size()) + " (" +
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index fa0736160df72..2b192a66c802f 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -62,6 +62,29 @@ class ConcatOutputSection : public OutputSection {
void finalizeFlags(InputSection *input);
};
+// We maintain one ThunkInfo per real function.
+//
+// The "active thunk" is represented by the sym/isec pair that
+// turns-over during finalize(): as the call-site address advances,
+// the active thunk goes out of branch-range, and we create a new
+// thunk to take its place.
+//
+// The remaining members -- bools and counters -- apply to the
+// collection of thunks associated with the real function.
+
+struct ThunkInfo {
+ // These denote the active thunk:
+ Defined *sym = nullptr; // private-extern symbol for active thunk
+ ConcatInputSection *isec = nullptr; // input section for active thunk
+
+ /// Before this thunk is created, this contains the set of branches with the
+ /// same target that could trigger this thunk's creation.
+ llvm::DenseSet<const Relocation *> pendingBranches;
+
+ // The following value is cumulative across all thunks on this function
+ uint8_t sequence = 0; // how many thunks created so-far?
+};
+
// ConcatOutputSections that contain code (text) require special handling to
// support thunk insertion.
class TextOutputSection : public ConcatOutputSection {
@@ -98,29 +121,14 @@ class TextOutputSection : public ConcatOutputSection {
bool isTargetStubsAndInRange(const ConcatInputSection &isec,
const Relocation &r,
uint64_t estimatedStubsEnd) const;
+ /// Mark the branch at \p r as resolved and possibly decrement
+ /// numPendingThunkTargets.
+ void markBranchAsResolved(ThunkInfo &thunkInfo, const Relocation *r);
/// The number of relocations updated to point to thunks.
size_t thunkCallCount = 0;
-};
-
-// We maintain one ThunkInfo per real function.
-//
-// The "active thunk" is represented by the sym/isec pair that
-// turns-over during finalize(): as the call-site address advances,
-// the active thunk goes out of branch-range, and we create a new
-// thunk to take its place.
-//
-// The remaining members -- bools and counters -- apply to the
-// collection of thunks associated with the real function.
-
-struct ThunkInfo {
- // These denote the active thunk:
- Defined *sym = nullptr; // private-extern symbol for active thunk
- ConcatInputSection *isec = nullptr; // input section for active thunk
-
- llvm::DenseSet<const Relocation *> pendingBranches;
-
- // The following value is cumulative across all thunks on this function
- uint8_t sequence = 0; // how many thunks created so-far?
+ /// The number of new thunks that could be created from our current list of
+ /// pending branches.
+ unsigned numPendingThunkTargets = 0;
};
NamePair maybeRenameSection(NamePair key);
>From 1b5874bf9d4e7e62d10ea62a1d2fab87eda40826 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Wed, 29 Apr 2026 17:59:47 -0700
Subject: [PATCH 14/17] call markBranchAsResolved in more places
---
lld/MachO/ConcatOutputSection.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 3bf92807bfb09..454ccf117a8c7 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -245,6 +245,7 @@ void TextOutputSection::finalize() {
}
if (auto *thunk = getThunkInRange(*callerIsec, *r)) {
deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
+ markBranchAsResolved(thunkInfo, r);
branchesToProcess.pop_front();
continue;
}
@@ -303,10 +304,10 @@ void TextOutputSection::finalize() {
llvm::erase_if(branchesToProcess, [&](auto &tuple) {
auto [callerIsec, r, thunkKey] = tuple;
- if (!isTargetKnownInRange(*callerIsec, *r))
- return false;
- markBranchAsResolved(thunkMap[thunkKey], r);
- return true;
+ bool targetInRange = isTargetKnownInRange(*callerIsec, *r);
+ if (targetInRange || getThunkInRange(*callerIsec, *r))
+ markBranchAsResolved(thunkMap[thunkKey], r);
+ return targetInRange;
});
#ifndef NDEBUG
>From 0c550bb6aa8447f1099ecba0e1efb8e2cbc0a40f Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Thu, 30 Apr 2026 16:51:02 -0700
Subject: [PATCH 15/17] avoid looking up key in thunkMap
I think I was hitting a bug where a value in thunkMap was becoming
invalidated because I was using thunkMap[] in a helper function. Avoid
that bug by passing thunkInfo directly in those helper functions to make
it more clear where we do thunkMap[]
---
lld/MachO/ConcatOutputSection.cpp | 55 ++++++++++++++++---------------
lld/MachO/ConcatOutputSection.h | 45 ++++++++++++-------------
2 files changed, 52 insertions(+), 48 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index f62eebc744551..a065e58989f5b 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -120,19 +120,16 @@ bool TextOutputSection::isTargetKnownInRange(const ConcatInputSection &isec,
}
Defined *TextOutputSection::getThunkInRange(const ConcatInputSection &isec,
- const Relocation &r) const {
+ const Relocation &r,
+ const ThunkInfo &thunkInfo) const {
assert(!isTargetKnownInRange(isec, r));
+ if (!thunkInfo.sym)
+ return nullptr;
uint64_t callVA = isec.getVA() + r.offset;
uint64_t lowVA = target->backwardBranchRange < callVA
? callVA - target->backwardBranchRange
: 0;
uint64_t highVA = callVA + target->forwardBranchRange;
- auto *funcSym = cast<Symbol *>(r.referent);
- auto it = thunkMap.find(ThunkKey{funcSym, r.addend});
- if (it == thunkMap.end())
- return nullptr;
- auto &thunkInfo = it->second;
- assert(thunkInfo.sym);
uint64_t thunkVA = thunkInfo.isec->getVA();
if (lowVA <= thunkVA && thunkVA <= highVA)
return thunkInfo.sym;
@@ -149,8 +146,8 @@ void TextOutputSection::updateBranchTargetToThunk(Relocation &r,
}
void TextOutputSection::createThunk(const ConcatInputSection &isec,
- Relocation &r) {
- assert(getThunkInRange(isec, r) == nullptr);
+ Relocation &r, ThunkInfo &thunkInfo) {
+ assert(getThunkInRange(isec, r, thunkInfo) == nullptr);
assert(isec.isFinal);
uint64_t highVA = isec.getVA() + r.offset + target->forwardBranchRange;
if (addr + size > highVA) {
@@ -162,8 +159,6 @@ void TextOutputSection::createThunk(const ConcatInputSection &isec,
": FIXME: thunk range overrun. Consider increasing the "
"slop-scale with `--slop-scale=<unsigned_int>`.");
}
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
thunkInfo.isec = makeSyntheticInputSection(isec.getSegName(), isec.getName());
thunkInfo.isec->parent = this;
assert(thunkInfo.isec->live);
@@ -172,6 +167,7 @@ void TextOutputSection::createThunk(const ConcatInputSection &isec,
if (r.addend != 0)
addendSuffix = "+" + std::to_string(r.addend);
size_t thunkSize = target->thunkSize;
+ auto *funcSym = cast<Symbol *>(r.referent);
StringRef thunkName =
saver().save(funcSym->getName() + addendSuffix + ".thunk." +
std::to_string(thunkInfo.sequence++));
@@ -217,8 +213,7 @@ void TextOutputSection::finalize() {
// Branches whose target sections are out of range or have not yet been
// finalized. We may need to emit thunks for them.
- std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
- branchesToProcess;
+ std::deque<std::pair<ConcatInputSection *, Relocation *>> branchesToProcess;
// Branches whose targets have not yet be finalized, but a thunk for that
// target exists. We defer processing these branches because it's possible we
// can still direct call to their targets after they have all been finalized.
@@ -228,13 +223,15 @@ void TextOutputSection::finalize() {
const uint64_t slop = config->slopScale * target->thunkSize;
for (auto *isec : inputs) {
while (!branchesToProcess.empty()) {
- auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
+ auto &[callerIsec, r] = branchesToProcess.front();
assert(callerIsec->isFinal);
+ auto *funcSym = cast<Symbol *>(r->referent);
+ auto &thunkInfo = thunkMap[{funcSym, r->addend}];
if (isTargetKnownInRange(*callerIsec, *r)) {
branchesToProcess.pop_front();
continue;
}
- if (auto *thunk = getThunkInRange(*callerIsec, *r)) {
+ if (auto *thunk = getThunkInRange(*callerIsec, *r, thunkInfo)) {
deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
branchesToProcess.pop_front();
continue;
@@ -250,7 +247,7 @@ void TextOutputSection::finalize() {
if (nextEnd + slop <= highVA)
break;
- createThunk(*callerIsec, *r);
+ createThunk(*callerIsec, *r, thunkInfo);
branchesToProcess.pop_front();
}
finalizeOne(isec);
@@ -274,27 +271,31 @@ void TextOutputSection::finalize() {
continue;
if (isTargetKnownInRange(*isec, r))
continue;
- if (auto *thunk = getThunkInRange(*isec, r)) {
+ auto *funcSym = cast<Symbol *>(r.referent);
+ auto &thunkInfo = thunkMap[{funcSym, r.addend}];
+ if (auto *thunk = getThunkInRange(*isec, r, thunkInfo)) {
deferredBranchRedirects.emplace_back(isec, &r, thunk);
continue;
}
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkKey key{funcSym, r.addend};
- branchesToProcess.emplace_back(isec, &r, key);
+ branchesToProcess.emplace_back(isec, &r);
}
}
llvm::erase_if(branchesToProcess, [&](auto &tuple) {
- auto [callerIsec, r, thunkKey] = tuple;
+ auto [callerIsec, r] = tuple;
return isTargetKnownInRange(*callerIsec, *r);
});
// Count distinct unresolved branch targets that still lack an in-range thunk.
// We use this as an upper bound on the number of thunks we may still create
// when estimating where __stubs / __objc_stubs could end up.
DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
- for (auto [callerIsec, r, thunkKey] : branchesToProcess)
- if (!getThunkInRange(*callerIsec, *r))
+ for (auto [callerIsec, r] : branchesToProcess) {
+ auto *funcSym = cast<Symbol *>(r->referent);
+ ThunkKey thunkKey{funcSym, r->addend};
+ auto &thunkInfo = thunkMap[thunkKey];
+ if (!getThunkInRange(*callerIsec, *r, thunkInfo))
branchTargets.insert(thunkKey);
+ }
uint64_t estimatedTextEnd =
addr + size + branchTargets.size() * target->thunkSize;
@@ -313,14 +314,16 @@ void TextOutputSection::finalize() {
updateBranchTargetToThunk(*r, thunk);
}
- for (auto [isec, r, thunkKey] : branchesToProcess) {
+ for (auto [isec, r] : branchesToProcess) {
if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
- if (auto *thunk = getThunkInRange(*isec, *r)) {
+ auto *funcSym = cast<Symbol *>(r->referent);
+ auto &thunkInfo = thunkMap[{funcSym, r->addend}];
+ if (auto *thunk = getThunkInRange(*isec, *r, thunkInfo)) {
updateBranchTargetToThunk(*r, thunk);
continue;
}
- createThunk(*isec, *r);
+ createThunk(*isec, *r, thunkInfo);
}
if (!thunks.empty())
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index 5693b1416e061..7de730fdb36e6 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -62,6 +62,25 @@ class ConcatOutputSection : public OutputSection {
void finalizeFlags(InputSection *input);
};
+// We maintain one ThunkInfo per real function.
+//
+// The "active thunk" is represented by the sym/isec pair that
+// turns-over during finalize(): as the call-site address advances,
+// the active thunk goes out of branch-range, and we create a new
+// thunk to take its place.
+//
+// The remaining members -- bools and counters -- apply to the
+// collection of thunks associated with the real function.
+
+struct ThunkInfo {
+ // These denote the active thunk:
+ Defined *sym = nullptr; // private-extern symbol for active thunk
+ ConcatInputSection *isec = nullptr; // input section for active thunk
+
+ // The following value is cumulative across all thunks on this function
+ uint8_t sequence = 0; // how many thunks created so-far?
+};
+
// ConcatOutputSections that contain code (text) require special handling to
// support thunk insertion.
class TextOutputSection : public ConcatOutputSection {
@@ -86,12 +105,13 @@ class TextOutputSection : public ConcatOutputSection {
const Relocation &r) const;
/// If there exists a thunk in range of the target in \p r, \return that
/// thunk.
- Defined *getThunkInRange(const ConcatInputSection &isec,
- const Relocation &r) const;
+ Defined *getThunkInRange(const ConcatInputSection &isec, const Relocation &r,
+ const ThunkInfo &thunkInfo) const;
/// Update \p r to target \p thunk which is guaranteed to be in range.
void updateBranchTargetToThunk(Relocation &r, Defined *thunk);
/// Create a new thunk and update \p r to target the new thunk.
- void createThunk(const ConcatInputSection &isec, Relocation &r);
+ void createThunk(const ConcatInputSection &isec, Relocation &r,
+ ThunkInfo &thunkInfo);
/// \return true if the target in \p r is in __stubs or __objc_stubs and in
/// range from the location in \p isec. \p estimatedStubsEnd is the estimated
/// VA of the end of the last stubs section.
@@ -102,25 +122,6 @@ class TextOutputSection : public ConcatOutputSection {
size_t thunkCallCount = 0;
};
-// We maintain one ThunkInfo per real function.
-//
-// The "active thunk" is represented by the sym/isec pair that
-// turns-over during finalize(): as the call-site address advances,
-// the active thunk goes out of branch-range, and we create a new
-// thunk to take its place.
-//
-// The remaining members -- bools and counters -- apply to the
-// collection of thunks associated with the real function.
-
-struct ThunkInfo {
- // These denote the active thunk:
- Defined *sym = nullptr; // private-extern symbol for active thunk
- ConcatInputSection *isec = nullptr; // input section for active thunk
-
- // The following value is cumulative across all thunks on this function
- uint8_t sequence = 0; // how many thunks created so-far?
-};
-
NamePair maybeRenameSection(NamePair key);
// Output sections are added to output segments in iteration order
>From e4c72c78233d3ab4de38b39975b4f51b08721615 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Thu, 30 Apr 2026 16:51:02 -0700
Subject: [PATCH 16/17] avoid looking up key in thunkMap
I think I was hitting a bug where a value in thunkMap was becoming
invalidated because I was using thunkMap[] in a helper function. Avoid
that bug by passing thunkInfo directly in those helper functions to make
it more clear where we do thunkMap[]
---
lld/MachO/ConcatOutputSection.cpp | 63 ++++++++++++++++---------------
lld/MachO/ConcatOutputSection.h | 45 +++++++++++-----------
2 files changed, 56 insertions(+), 52 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index f62eebc744551..50c5c607f8234 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -120,19 +120,16 @@ bool TextOutputSection::isTargetKnownInRange(const ConcatInputSection &isec,
}
Defined *TextOutputSection::getThunkInRange(const ConcatInputSection &isec,
- const Relocation &r) const {
+ const Relocation &r,
+ const ThunkInfo &thunkInfo) const {
assert(!isTargetKnownInRange(isec, r));
+ if (!thunkInfo.sym)
+ return nullptr;
uint64_t callVA = isec.getVA() + r.offset;
uint64_t lowVA = target->backwardBranchRange < callVA
? callVA - target->backwardBranchRange
: 0;
uint64_t highVA = callVA + target->forwardBranchRange;
- auto *funcSym = cast<Symbol *>(r.referent);
- auto it = thunkMap.find(ThunkKey{funcSym, r.addend});
- if (it == thunkMap.end())
- return nullptr;
- auto &thunkInfo = it->second;
- assert(thunkInfo.sym);
uint64_t thunkVA = thunkInfo.isec->getVA();
if (lowVA <= thunkVA && thunkVA <= highVA)
return thunkInfo.sym;
@@ -149,8 +146,8 @@ void TextOutputSection::updateBranchTargetToThunk(Relocation &r,
}
void TextOutputSection::createThunk(const ConcatInputSection &isec,
- Relocation &r) {
- assert(getThunkInRange(isec, r) == nullptr);
+ Relocation &r, ThunkInfo &thunkInfo) {
+ assert(getThunkInRange(isec, r, thunkInfo) == nullptr);
assert(isec.isFinal);
uint64_t highVA = isec.getVA() + r.offset + target->forwardBranchRange;
if (addr + size > highVA) {
@@ -162,8 +159,6 @@ void TextOutputSection::createThunk(const ConcatInputSection &isec,
": FIXME: thunk range overrun. Consider increasing the "
"slop-scale with `--slop-scale=<unsigned_int>`.");
}
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkInfo &thunkInfo = thunkMap[ThunkKey{funcSym, r.addend}];
thunkInfo.isec = makeSyntheticInputSection(isec.getSegName(), isec.getName());
thunkInfo.isec->parent = this;
assert(thunkInfo.isec->live);
@@ -172,6 +167,7 @@ void TextOutputSection::createThunk(const ConcatInputSection &isec,
if (r.addend != 0)
addendSuffix = "+" + std::to_string(r.addend);
size_t thunkSize = target->thunkSize;
+ auto *funcSym = cast<Symbol *>(r.referent);
StringRef thunkName =
saver().save(funcSym->getName() + addendSuffix + ".thunk." +
std::to_string(thunkInfo.sequence++));
@@ -217,8 +213,7 @@ void TextOutputSection::finalize() {
// Branches whose target sections are out of range or have not yet been
// finalized. We may need to emit thunks for them.
- std::deque<std::tuple<ConcatInputSection *, Relocation *, ThunkKey>>
- branchesToProcess;
+ std::deque<std::pair<ConcatInputSection *, Relocation *>> branchesToProcess;
// Branches whose targets have not yet be finalized, but a thunk for that
// target exists. We defer processing these branches because it's possible we
// can still direct call to their targets after they have all been finalized.
@@ -228,13 +223,15 @@ void TextOutputSection::finalize() {
const uint64_t slop = config->slopScale * target->thunkSize;
for (auto *isec : inputs) {
while (!branchesToProcess.empty()) {
- auto &[callerIsec, r, thunkKey] = branchesToProcess.front();
+ auto [callerIsec, r] = branchesToProcess.front();
assert(callerIsec->isFinal);
+ auto *funcSym = cast<Symbol *>(r->referent);
+ auto &thunkInfo = thunkMap[{funcSym, r->addend}];
if (isTargetKnownInRange(*callerIsec, *r)) {
branchesToProcess.pop_front();
continue;
}
- if (auto *thunk = getThunkInRange(*callerIsec, *r)) {
+ if (auto *thunk = getThunkInRange(*callerIsec, *r, thunkInfo)) {
deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
branchesToProcess.pop_front();
continue;
@@ -250,7 +247,7 @@ void TextOutputSection::finalize() {
if (nextEnd + slop <= highVA)
break;
- createThunk(*callerIsec, *r);
+ createThunk(*callerIsec, *r, thunkInfo);
branchesToProcess.pop_front();
}
finalizeOne(isec);
@@ -274,27 +271,31 @@ void TextOutputSection::finalize() {
continue;
if (isTargetKnownInRange(*isec, r))
continue;
- if (auto *thunk = getThunkInRange(*isec, r)) {
+ auto *funcSym = cast<Symbol *>(r.referent);
+ auto &thunkInfo = thunkMap[{funcSym, r.addend}];
+ if (auto *thunk = getThunkInRange(*isec, r, thunkInfo)) {
deferredBranchRedirects.emplace_back(isec, &r, thunk);
continue;
}
- auto *funcSym = cast<Symbol *>(r.referent);
- ThunkKey key{funcSym, r.addend};
- branchesToProcess.emplace_back(isec, &r, key);
+ branchesToProcess.emplace_back(isec, &r);
}
}
- llvm::erase_if(branchesToProcess, [&](auto &tuple) {
- auto [callerIsec, r, thunkKey] = tuple;
+ llvm::erase_if(branchesToProcess, [&](auto &pair) {
+ auto [callerIsec, r] = pair;
return isTargetKnownInRange(*callerIsec, *r);
});
// Count distinct unresolved branch targets that still lack an in-range thunk.
// We use this as an upper bound on the number of thunks we may still create
// when estimating where __stubs / __objc_stubs could end up.
DenseSet<ThunkKey, ThunkMapKeyInfo> branchTargets;
- for (auto [callerIsec, r, thunkKey] : branchesToProcess)
- if (!getThunkInRange(*callerIsec, *r))
+ for (auto [callerIsec, r] : branchesToProcess) {
+ auto *funcSym = cast<Symbol *>(r->referent);
+ ThunkKey thunkKey{funcSym, r->addend};
+ auto &thunkInfo = thunkMap[thunkKey];
+ if (!getThunkInRange(*callerIsec, *r, thunkInfo))
branchTargets.insert(thunkKey);
+ }
uint64_t estimatedTextEnd =
addr + size + branchTargets.size() * target->thunkSize;
@@ -305,22 +306,24 @@ void TextOutputSection::finalize() {
alignToPowerOf2(estimatedStubsEnd, in.objcStubs->align) +
in.objcStubs->getSize();
- for (auto [callerIsec, r, thunk] : deferredBranchRedirects) {
- if (isTargetKnownInRange(*callerIsec, *r))
+ for (auto [isec, r, thunk] : deferredBranchRedirects) {
+ if (isTargetKnownInRange(*isec, *r))
continue;
- if (isTargetStubsAndInRange(*callerIsec, *r, estimatedStubsEnd))
+ if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
updateBranchTargetToThunk(*r, thunk);
}
- for (auto [isec, r, thunkKey] : branchesToProcess) {
+ for (auto [isec, r] : branchesToProcess) {
if (isTargetStubsAndInRange(*isec, *r, estimatedStubsEnd))
continue;
- if (auto *thunk = getThunkInRange(*isec, *r)) {
+ auto *funcSym = cast<Symbol *>(r->referent);
+ auto &thunkInfo = thunkMap[{funcSym, r->addend}];
+ if (auto *thunk = getThunkInRange(*isec, *r, thunkInfo)) {
updateBranchTargetToThunk(*r, thunk);
continue;
}
- createThunk(*isec, *r);
+ createThunk(*isec, *r, thunkInfo);
}
if (!thunks.empty())
diff --git a/lld/MachO/ConcatOutputSection.h b/lld/MachO/ConcatOutputSection.h
index 5693b1416e061..7de730fdb36e6 100644
--- a/lld/MachO/ConcatOutputSection.h
+++ b/lld/MachO/ConcatOutputSection.h
@@ -62,6 +62,25 @@ class ConcatOutputSection : public OutputSection {
void finalizeFlags(InputSection *input);
};
+// We maintain one ThunkInfo per real function.
+//
+// The "active thunk" is represented by the sym/isec pair that
+// turns-over during finalize(): as the call-site address advances,
+// the active thunk goes out of branch-range, and we create a new
+// thunk to take its place.
+//
+// The remaining members -- bools and counters -- apply to the
+// collection of thunks associated with the real function.
+
+struct ThunkInfo {
+ // These denote the active thunk:
+ Defined *sym = nullptr; // private-extern symbol for active thunk
+ ConcatInputSection *isec = nullptr; // input section for active thunk
+
+ // The following value is cumulative across all thunks on this function
+ uint8_t sequence = 0; // how many thunks created so-far?
+};
+
// ConcatOutputSections that contain code (text) require special handling to
// support thunk insertion.
class TextOutputSection : public ConcatOutputSection {
@@ -86,12 +105,13 @@ class TextOutputSection : public ConcatOutputSection {
const Relocation &r) const;
/// If there exists a thunk in range of the target in \p r, \return that
/// thunk.
- Defined *getThunkInRange(const ConcatInputSection &isec,
- const Relocation &r) const;
+ Defined *getThunkInRange(const ConcatInputSection &isec, const Relocation &r,
+ const ThunkInfo &thunkInfo) const;
/// Update \p r to target \p thunk which is guaranteed to be in range.
void updateBranchTargetToThunk(Relocation &r, Defined *thunk);
/// Create a new thunk and update \p r to target the new thunk.
- void createThunk(const ConcatInputSection &isec, Relocation &r);
+ void createThunk(const ConcatInputSection &isec, Relocation &r,
+ ThunkInfo &thunkInfo);
/// \return true if the target in \p r is in __stubs or __objc_stubs and in
/// range from the location in \p isec. \p estimatedStubsEnd is the estimated
/// VA of the end of the last stubs section.
@@ -102,25 +122,6 @@ class TextOutputSection : public ConcatOutputSection {
size_t thunkCallCount = 0;
};
-// We maintain one ThunkInfo per real function.
-//
-// The "active thunk" is represented by the sym/isec pair that
-// turns-over during finalize(): as the call-site address advances,
-// the active thunk goes out of branch-range, and we create a new
-// thunk to take its place.
-//
-// The remaining members -- bools and counters -- apply to the
-// collection of thunks associated with the real function.
-
-struct ThunkInfo {
- // These denote the active thunk:
- Defined *sym = nullptr; // private-extern symbol for active thunk
- ConcatInputSection *isec = nullptr; // input section for active thunk
-
- // The following value is cumulative across all thunks on this function
- uint8_t sequence = 0; // how many thunks created so-far?
-};
-
NamePair maybeRenameSection(NamePair key);
// Output sections are added to output segments in iteration order
>From 8fec6f4c49f2ee4a05adf7789119761d2b46560f Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Fri, 1 May 2026 17:02:56 -0700
Subject: [PATCH 17/17] Avoid calling markBranchAsResolved() when we discover a
thunk in range
The only way for a thunk to be in range is if we created one. In that
case, we already call thunkInfo.pendingBranches.clear() so there is no
need to remove this branch.
---
lld/MachO/ConcatOutputSection.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index f54715dfb2bfc..2977b9907daef 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -243,7 +243,6 @@ void TextOutputSection::finalize() {
}
if (auto *thunk = getThunkInRange(*callerIsec, *r, thunkInfo)) {
deferredBranchRedirects.emplace_back(callerIsec, r, thunk);
- markBranchAsResolved(r, thunkInfo);
branchesToProcess.pop_front();
continue;
}
@@ -301,12 +300,12 @@ void TextOutputSection::finalize() {
llvm::erase_if(branchesToProcess, [&](auto &pair) {
auto [callerIsec, r] = pair;
- bool targetInRange = isTargetKnownInRange(*callerIsec, *r);
+ if (!isTargetKnownInRange(*callerIsec, *r))
+ return false;
auto *funcSym = cast<Symbol *>(r->referent);
auto &thunkInfo = thunkMap[{funcSym, r->addend}];
- if (targetInRange || getThunkInRange(*callerIsec, *r, thunkInfo))
- markBranchAsResolved(r, thunkInfo);
- return targetInRange;
+ markBranchAsResolved(r, thunkInfo);
+ return true;
});
#ifndef NDEBUG
More information about the llvm-commits
mailing list