[lld] [ELF] Add -z sort-thunks to sort thunks by destination (PR #211721)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 25 16:06:57 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/211721
>From f16377deb7c1a42ce898382702bc6edbe1676b41 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 23 Jul 2026 21:05:31 -0700
Subject: [PATCH] [ELF] Add -z sort-thunks to sort thunks by destination
A thunk's distance to a forward destination grows when a thunk after it
grows.
For
```
[forward A B C] dstA dstB dstC
```
When all thunks are initially short, Promoting C can push B to be
enlarged on the following pass, which in turn push A to be promoted on
the following pass. Promoting just one thunk on a pass can therefore
trigger "address assignment did not converge".
Add -z sort-thunks (off by default) to sort each ThunkSection once pass
0 has created its thunks. A forward thunk's distance grows only when a
thunk after it grows, so order forward thunks by descending destination.
A backward thunk's distance grows only with a promotion before it,
already applied when we reach it, so backward thunks need no ordering;
partition them ahead of the forward thunks in creation order.
Prefix ThunkSections (MIPS LA25, AArch64 BTI landing pads) hold
alternative entry points whose order is meaningful and are left unsorted.
Co-authored-by: Pranav Kant <prka at google.com>
---
lld/ELF/Config.h | 1 +
lld/ELF/Driver.cpp | 1 +
lld/ELF/Relocations.cpp | 10 ++-
lld/ELF/SyntheticSections.cpp | 26 ++++++++
lld/ELF/SyntheticSections.h | 4 ++
lld/ELF/Thunks.cpp | 5 ++
lld/ELF/Thunks.h | 2 +
lld/test/ELF/aarch64-thunk-sort.s | 102 ++++++++++++++++++++++++++++++
8 files changed, 150 insertions(+), 1 deletion(-)
create mode 100644 lld/test/ELF/aarch64-thunk-sort.s
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 1426aab12758f..bd74699eda235 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -452,6 +452,7 @@ struct Config {
bool zRodynamic;
bool zSectionHeader;
bool zShstk;
+ bool zSortThunks;
bool zStartStopGC;
uint8_t zStartStopVisibility;
bool zText;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 51d9e32419f9f..8f0e8ecfd936c 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1678,6 +1678,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
ctx.arg.zRodynamic = hasZOption(args, "rodynamic");
ctx.arg.zSeparate = getZSeparate(args);
ctx.arg.zShstk = hasZOption(args, "shstk");
+ ctx.arg.zSortThunks = getZFlag(args, "sort-thunks", "nosort-thunks", false);
ctx.arg.zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0);
ctx.arg.zStartStopGC =
getZFlag(args, "start-stop-gc", "nostart-stop-gc", true);
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 4702d941d28ca..7496c86ce5e18 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1741,6 +1741,7 @@ ThunkSection *ThunkCreator::addThunkSection(OutputSection *os,
uint64_t off, bool isPrefix) {
auto *ts = make<ThunkSection>(ctx, os, off);
ts->partition = os->partition;
+ ts->isPrefix = isPrefix;
if ((ctx.arg.fixCortexA53Errata843419 || ctx.arg.fixCortexA8) &&
!isd->sections.empty() && !isPrefix) {
// The errata fixes are sensitive to addresses modulo 4 KiB. When we add
@@ -1966,8 +1967,15 @@ bool ThunkCreator::createThunks(uint32_t pass,
rel.addend = -getPCBias(ctx, *isec, rel);
}
- for (auto &p : isd->thunkSections)
+ for (auto &p : isd->thunkSections) {
+ // Sort in pass 0, which creates most thunks. Sorting in later passes
+ // could oscillate: the sort keys are derived from the very layout
+ // being recomputed. Prefix thunk sections hold alternative entry
+ // points whose order is meaningful and must not be reordered.
+ if (pass == 0 && ctx.arg.zSortThunks && !p.first->isPrefix)
+ p.first->sortByDestination();
addressesChanged |= p.first->assignOffsets();
+ }
});
for (auto &p : thunkedSections)
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index fda7a835827ce..325b0eea27e24 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -4107,6 +4107,32 @@ InputSection *ThunkSection::getTargetInputSection() const {
return t->getTargetInputSection();
}
+// Move forward thunks to the right half and sort them by destination VA:
+//
+// dstA, dstB, [backward A, B], [forward D, C], dstC, dstD
+//
+// A forward thunk's distance grows when a thunk after it grows. Ordering
+// forward thunks by descending destination keeps the most promotable ones
+// lowest, where their growth stays below the rest. A backward thunk's distance
+// grows only with a promotion before it, already applied when we reach it, so
+// backward thunks need no ordering and stay ahead of forward thunks in creation
+// order.
+void ThunkSection::sortByDestination() {
+ uint64_t base = getVA();
+ SmallVector<std::pair<uint64_t, Thunk *>, 0> keys;
+ keys.resize_for_overwrite(thunks.size());
+ for (auto [i, t] : enumerate(thunks))
+ keys[i] = {t->getDestVA(), t};
+ auto *forward =
+ std::stable_partition(keys.begin(), keys.end(),
+ [base](const auto &k) { return k.first <= base; });
+ std::stable_sort(forward, keys.end(), [](const auto &a, const auto &b) {
+ return a.first > b.first;
+ });
+ for (auto [i, p] : llvm::enumerate(keys))
+ thunks[i] = p.second;
+}
+
bool ThunkSection::assignOffsets() {
uint64_t off = 0;
bool changed = false;
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 6e7df2461ba35..e1ba67bced5f2 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -1236,11 +1236,15 @@ class ThunkSection final : public SyntheticSection {
void writeTo(uint8_t *buf) override;
InputSection *getTargetInputSection() const;
bool assignOffsets();
+ void sortByDestination();
// When true, round up reported size of section to 4 KiB. See comment
// in addThunkSection() for more details.
bool roundUpSizeForErrata = false;
+ // True for a ThunkSection placed immediately before its target section.
+ bool isPrefix = false;
+
private:
SmallVector<Thunk *, 0> thunks;
size_t size = 0;
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 1f161685f178d..c8f01b0e004d0 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -629,6 +629,11 @@ void Thunk::setOffset(uint64_t newOffset) {
offset = newOffset;
}
+uint64_t Thunk::getDestVA() const {
+ return destination.isInPlt(ctx) ? destination.getPltVA(ctx)
+ : destination.getVA(ctx, addend);
+}
+
// AArch64 Thunk base class.
static uint64_t getAArch64ThunkDestVA(Ctx &ctx, const Symbol &s, int64_t a) {
uint64_t v = s.isInPlt(ctx) ? s.getPltVA(ctx) : s.getVA(ctx, a);
diff --git a/lld/ELF/Thunks.h b/lld/ELF/Thunks.h
index 446345b8517f9..e99e21783b317 100644
--- a/lld/ELF/Thunks.h
+++ b/lld/ELF/Thunks.h
@@ -60,6 +60,8 @@ class Thunk {
// enabled.
virtual bool needsSyntheticLandingPad() { return false; }
+ uint64_t getDestVA() const;
+
Defined *getThunkTargetSym() const { return syms[0]; }
Ctx &ctx;
diff --git a/lld/test/ELF/aarch64-thunk-sort.s b/lld/test/ELF/aarch64-thunk-sort.s
new file mode 100644
index 0000000000000..4f845f7305eca
--- /dev/null
+++ b/lld/test/ELF/aarch64-thunk-sort.s
@@ -0,0 +1,102 @@
+# REQUIRES: aarch64
+## -z sort-thunks orders a thunk section's forward thunks by descending destination so promoting
+## one to its long form cannot push another out of range. In creation order the promotions cascade,
+## one per pass, and exceed convergence limit (issue #61250).
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 a.s -o a.o
+# RUN: ld.lld -T lds a.o -z sort-thunks -o out
+# RUN: llvm-objdump -d --no-show-raw-insn out | FileCheck %s
+## The default keeps creation order and does not converge.
+# RUN: not ld.lld -T lds a.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+# ERR: error: address assignment did not converge
+
+## One thunk section holds both directions: backward thunks first, in creation
+## order lo0, lo1, lo2 (they need no sorting), then forward thunks by descending
+## destination. The farthest forward thunks become long (ldr), nearer ones short (b).
+# CHECK: <__AArch64AbsLongThunk_lo0>:
+# CHECK: <__AArch64AbsLongThunk_lo1>:
+# CHECK: <__AArch64AbsLongThunk_lo2>:
+# CHECK: <__AArch64AbsLongThunk_hi43>:
+# CHECK-NEXT: ldr x16,
+# CHECK: <__AArch64AbsLongThunk_hi35>:
+# CHECK-NEXT: ldr x16,
+# CHECK: <__AArch64AbsLongThunk_hi34>:
+# CHECK-NEXT: b {{.*}} <hi34>
+# CHECK: <__AArch64AbsLongThunk_hi0>:
+
+## getDestVA keys a PLT-routed thunk on the PLT entry, not the symbol value. bar's thunk sorts after
+## the backward thunk to local cbwd; the symbol value 0 would place it backward and flip the order.
+# RUN: llvm-mc -filetype=obj -triple=aarch64 b.s -o b.o
+# RUN: ld.lld -shared b.o -o b.so
+# RUN: llvm-mc -filetype=obj -triple=aarch64 c.s -o c.o
+# RUN: ld.lld -T lds2 c.o b.so -z sort-thunks -o out2
+# RUN: llvm-objdump -d --no-show-raw-insn out2 | FileCheck %s --check-prefix=PLT
+# PLT: <__AArch64AbsLongThunk_cbwd>:
+# PLT: <__AArch64AbsLongThunk_bar>:
+
+#--- a.s
+.section .low,"ax",%progbits
+.rept 3
+.globl lo\+
+lo\+:
+ ret
+ .space 12
+.endr
+
+.text
+.globl _start
+_start:
+.rept 3
+ bl lo\+
+.endr
+.rept 44
+ bl hi\+
+ .space 12
+.endr
+
+.section .high,"ax",%progbits
+.rept 44
+ .globl hi\+
+hi\+:
+ ret
+ .space 12
+.endr
+
+#--- lds
+## .low sits far below .text, so lo<n> calls need (always long) backward thunks.
+## .high follows a gap tuned (688 = 44*16 - 16 + 12) so that, once the thunk
+## section shifts .high, only the farthest hi<i> is out of range; in creation
+## order each promotion then pushes out exactly one more.
+SECTIONS {
+ .low 0x10000 : { *(.low) }
+ .text 0x10000000 : { *(.text) }
+ . = . + 0x8000000 - 688;
+ .high : { *(.high) }
+}
+
+#--- b.s
+.globl bar
+bar:
+ ret
+
+#--- c.s
+.section .cbwd,"ax",%progbits
+.globl cbwd
+cbwd:
+ ret
+
+.text
+.globl _start
+_start:
+ bl bar
+ bl cbwd
+ ret
+
+#--- lds2
+## cbwd is far below and the PLT far above .text, so both calls need thunks.
+SECTIONS {
+ .cbwd 0x1000 : { *(.cbwd) }
+ .text 0x10000000 : { *(.text) }
+ .plt 0x20000000 : { *(.plt) }
+}
More information about the llvm-commits
mailing list