[lld] [lld] Generalize BP compression sort to support arbitrary section groups (PR #185661)
Zhaoxuan Jiang via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 10 07:38:02 PDT 2026
https://github.com/nocchijiang created https://github.com/llvm/llvm-project/pull/185661
Previously, BP compression sort used a single bucket for all code sections and a single bucket for all data sections. This patch introduces getCodeCompressionGroupId() and getDataCompressionGroupId() in the CRTP interface, allowing backends to partition sections into multiple groups that are optimized by BP independently and laid out in group-ID order.
For ELF, this separates .text.unlikely and .text.split from normal .text, so cold sections are grouped among themselves rather than mixed with hot code during compression optimization.
For MachO, both methods return 0 for now. Cold grouping via N_COLD_FUNC will be added in #183909.
>From 2935cb4ab7106f56cc4d3110613830bc58033446 Mon Sep 17 00:00:00 2001
From: Zhaoxuan Jiang <jiangzhaoxuan94 at gmail.com>
Date: Tue, 10 Mar 2026 22:02:20 +0800
Subject: [PATCH] [lld] Generalize BP compression sort to support arbitrary
section groups
Previously, BP compression sort used a single bucket for all code
sections and a single bucket for all data sections. This patch
introduces getCodeCompressionGroupId() and getDataCompressionGroupId()
in the CRTP interface, allowing backends to partition sections into
multiple groups that are optimized by BP independently and laid out in
group-ID order.
For ELF, this separates .text.unlikely and .text.split from normal
.text, so cold sections are grouped among themselves rather than mixed
with hot code during compression optimization.
For MachO, both methods return 0 for now. Cold grouping via
N_COLD_FUNC will be added in #183909.
---
lld/ELF/BPSectionOrderer.cpp | 8 +
lld/MachO/BPSectionOrderer.cpp | 3 +
.../lld/Common/BPSectionOrdererBase.inc | 188 ++++++++++--------
lld/test/ELF/bp-section-orderer-cold.s | 132 ++++++++++++
4 files changed, 251 insertions(+), 80 deletions(-)
create mode 100644 lld/test/ELF/bp-section-orderer-cold.s
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index 38f15b93050c9..2fd66b49d7eb5 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -32,6 +32,14 @@ struct BPOrdererELF : lld::BPOrderer<BPOrdererELF> {
static bool isCodeSection(const Section &sec) {
return sec.flags & ELF::SHF_EXECINSTR;
}
+ static unsigned getCodeCompressionGroupId(const Section &sec) {
+ if (sec.name.starts_with(".text.unlikely"))
+ return 1;
+ if (sec.name.starts_with(".text.split"))
+ return 2;
+ return 0;
+ }
+ static unsigned getDataCompressionGroupId(const Section &sec) { return 0; }
ArrayRef<Defined *> getSymbols(const Section &sec) {
auto it = secToSym.find(&sec);
if (it == secToSym.end())
diff --git a/lld/MachO/BPSectionOrderer.cpp b/lld/MachO/BPSectionOrderer.cpp
index c9c6c1c62bdf9..7ec32612ad764 100644
--- a/lld/MachO/BPSectionOrderer.cpp
+++ b/lld/MachO/BPSectionOrderer.cpp
@@ -34,6 +34,9 @@ struct BPOrdererMachO : lld::BPOrderer<BPOrdererMachO> {
static bool isCodeSection(const Section &sec) {
return macho::isCodeSection(&sec);
}
+ // TODO: Use N_COLD_FUNC to separate cold code into a different group.
+ static unsigned getCodeCompressionGroupId(const Section &sec) { return 0; }
+ static unsigned getDataCompressionGroupId(const Section &sec) { return 0; }
static ArrayRef<Defined *> getSymbols(const Section &sec) {
return sec.symbols;
}
diff --git a/lld/include/lld/Common/BPSectionOrdererBase.inc b/lld/include/lld/Common/BPSectionOrdererBase.inc
index 7d13cd25c0e76..f8f58102fad1c 100644
--- a/lld/include/lld/Common/BPSectionOrdererBase.inc
+++ b/lld/include/lld/Common/BPSectionOrdererBase.inc
@@ -221,20 +221,32 @@ auto BPOrderer<D>::computeOrder(
}
}
- SmallVector<unsigned> sectionIdxsForFunctionCompression,
- sectionIdxsForDataCompression;
+ // Collect sections into groups for compression. Each backend defines its own
+ // grouping via getCodeCompressionGroupId() and getDataCompressionGroupId().
+ // Each group is optimized by BP independently and laid out in group-ID order.
+ DenseMap<unsigned, SmallVector<unsigned>> codeGroupMap, dataGroupMap;
for (unsigned sectionIdx = 0; sectionIdx < sections.size(); sectionIdx++) {
if (startupSectionIdxUNs.contains(sectionIdx))
continue;
const auto *isec = sections[sectionIdx];
if (D::isCodeSection(*isec)) {
if (forFunctionCompression)
- sectionIdxsForFunctionCompression.push_back(sectionIdx);
+ codeGroupMap[D::getCodeCompressionGroupId(*isec)].push_back(sectionIdx);
} else {
if (forDataCompression)
- sectionIdxsForDataCompression.push_back(sectionIdx);
+ dataGroupMap[D::getDataCompressionGroupId(*isec)].push_back(sectionIdx);
}
}
+ auto getSortedGroupIds =
+ [](const DenseMap<unsigned, SmallVector<unsigned>> &m) {
+ SmallVector<unsigned> ids;
+ for (auto &[id, _] : m)
+ ids.push_back(id);
+ llvm::sort(ids);
+ return ids;
+ };
+ SmallVector<unsigned> codeGroupIds = getSortedGroupIds(codeGroupMap);
+ SmallVector<unsigned> dataGroupIds = getSortedGroupIds(dataGroupMap);
if (compressionSortStartupFunctions) {
SmallVector<unsigned> startupIdxs;
@@ -255,48 +267,81 @@ auto BPOrderer<D>::computeOrder(
// Map a section index (order directly) to a list of duplicate section indices
// (not ordered directly).
DenseMap<unsigned, SmallVector<unsigned, 0>> duplicateSectionIdxs;
- auto unsForFunctionCompression = getUnsForCompression<D>(
- sections, sectionToIdx, sectionIdxsForFunctionCompression,
- &duplicateSectionIdxs, maxUN);
- auto unsForDataCompression = getUnsForCompression<D>(
- sections, sectionToIdx, sectionIdxsForDataCompression,
- &duplicateSectionIdxs, maxUN);
-
- std::vector<BPFunctionNode> nodesForStartup, nodesForFunctionCompression,
- nodesForDataCompression;
+ auto buildGroupNodes =
+ [&](const SmallVector<unsigned> &groupIds,
+ DenseMap<unsigned, SmallVector<unsigned>> &groupMap) {
+ SmallVector<std::vector<BPFunctionNode>> groupNodes;
+ for (unsigned groupId : groupIds) {
+ auto uns =
+ getUnsForCompression<D>(sections, sectionToIdx, groupMap[groupId],
+ &duplicateSectionIdxs, maxUN);
+ std::vector<BPFunctionNode> nodes;
+ for (auto &[sectionIdx, un] : uns)
+ nodes.emplace_back(sectionIdx, un);
+ // Sort compression nodes by their Id (which is the section index)
+ // because the input linker order tends to be not bad.
+ llvm::sort(nodes, [](auto &L, auto &R) { return L.Id < R.Id; });
+ groupNodes.push_back(std::move(nodes));
+ }
+ return groupNodes;
+ };
+ SmallVector<std::vector<BPFunctionNode>> codeGroupNodes =
+ buildGroupNodes(codeGroupIds, codeGroupMap);
+ SmallVector<std::vector<BPFunctionNode>> dataGroupNodes =
+ buildGroupNodes(dataGroupIds, dataGroupMap);
+
+ std::vector<BPFunctionNode> nodesForStartup;
for (auto &[sectionIdx, uns] : startupSectionIdxUNs)
nodesForStartup.emplace_back(sectionIdx, uns);
- for (auto &[sectionIdx, uns] : unsForFunctionCompression)
- nodesForFunctionCompression.emplace_back(sectionIdx, uns);
- for (auto &[sectionIdx, uns] : unsForDataCompression)
- nodesForDataCompression.emplace_back(sectionIdx, uns);
// Use the first timestamp to define the initial order for startup nodes.
llvm::sort(nodesForStartup, [§ionIdxToTimestamp](auto &L, auto &R) {
return std::make_pair(sectionIdxToTimestamp[L.Id], L.Id) <
std::make_pair(sectionIdxToTimestamp[R.Id], R.Id);
});
- // Sort compression nodes by their Id (which is the section index) because the
- // input linker order tends to be not bad.
- llvm::sort(nodesForFunctionCompression,
- [](auto &L, auto &R) { return L.Id < R.Id; });
- llvm::sort(nodesForDataCompression,
- [](auto &L, auto &R) { return L.Id < R.Id; });
{
TimeTraceScope timeScope("Balanced Partitioning");
BalancedPartitioningConfig config;
BalancedPartitioning bp(config);
bp.run(nodesForStartup);
- bp.run(nodesForFunctionCompression);
- bp.run(nodesForDataCompression);
+ for (auto &nodes : codeGroupNodes)
+ bp.run(nodes);
+ for (auto &nodes : dataGroupNodes)
+ bp.run(nodes);
}
+ struct GroupStats {
+ unsigned numSections = 0, size = 0;
+ unsigned numDuplicates = 0, duplicateSize = 0;
+ };
+ auto collectGroupResults =
+ [&](SetVector<const Section *> &orderedSections,
+ SmallVector<std::vector<BPFunctionNode>> &groupNodes) {
+ SmallVector<GroupStats> stats(groupNodes.size());
+ for (auto [i, nodes] : llvm::enumerate(groupNodes)) {
+ for (auto &node : nodes) {
+ const auto *isec = sections[node.Id];
+ if (orderedSections.insert(isec)) {
+ stats[i].size += D::getSize(*isec);
+ ++stats[i].numSections;
+ }
+ auto It = duplicateSectionIdxs.find(node.Id);
+ if (It == duplicateSectionIdxs.end())
+ continue;
+ for (auto dupSecIdx : It->getSecond()) {
+ const auto *dupIsec = sections[dupSecIdx];
+ if (orderedSections.insert(dupIsec)) {
+ stats[i].duplicateSize += D::getSize(*dupIsec);
+ ++stats[i].numDuplicates;
+ }
+ }
+ }
+ }
+ return stats;
+ };
+
unsigned numStartupSections = 0, startupSize = 0;
- unsigned numCodeCompressionSections = 0, codeCompressionSize = 0;
- unsigned numDuplicateCodeSections = 0, duplicateCodeSize = 0;
- unsigned numDataCompressionSections = 0, dataCompressionSize = 0;
- unsigned numDuplicateDataSections = 0, duplicateDataSize = 0;
SetVector<const Section *> orderedSections;
// Order startup functions,
for (auto &node : nodesForStartup) {
@@ -306,63 +351,46 @@ auto BPOrderer<D>::computeOrder(
++numStartupSections;
}
}
- // then functions for compression,
- for (auto &node : nodesForFunctionCompression) {
- const auto *isec = sections[node.Id];
- if (orderedSections.insert(isec)) {
- codeCompressionSize += D::getSize(*isec);
- ++numCodeCompressionSections;
- }
- auto It = duplicateSectionIdxs.find(node.Id);
- if (It == duplicateSectionIdxs.end())
- continue;
- for (auto dupSecIdx : It->getSecond()) {
- const auto *dupIsec = sections[dupSecIdx];
- if (orderedSections.insert(dupIsec)) {
- duplicateCodeSize += D::getSize(*dupIsec);
- ++numDuplicateCodeSections;
- }
- }
- }
- // then data for compression.
- for (auto &node : nodesForDataCompression) {
- const auto *isec = sections[node.Id];
- if (orderedSections.insert(isec)) {
- dataCompressionSize += D::getSize(*isec);
- ++numDataCompressionSections;
- }
- auto It = duplicateSectionIdxs.find(node.Id);
- if (It == duplicateSectionIdxs.end())
- continue;
- for (auto dupSecIdx : It->getSecond()) {
- const auto *dupIsec = sections[dupSecIdx];
- if (orderedSections.insert(dupIsec)) {
- duplicateDataSize += D::getSize(*dupIsec);
- ++numDuplicateDataSections;
- }
- }
- }
+ // then code groups for compression,
+ SmallVector<GroupStats> codeStats =
+ collectGroupResults(orderedSections, codeGroupNodes);
+ // then data groups for compression.
+ SmallVector<GroupStats> dataStats =
+ collectGroupResults(orderedSections, dataGroupNodes);
if (verbose) {
- unsigned numTotalOrderedSections =
- numStartupSections + numCodeCompressionSections +
- numDuplicateCodeSections + numDataCompressionSections +
- numDuplicateDataSections;
- unsigned totalOrderedSize = startupSize + codeCompressionSize +
- duplicateCodeSize + dataCompressionSize +
- duplicateDataSize;
+ unsigned numTotalOrderedSections = numStartupSections;
+ unsigned totalOrderedSize = startupSize;
+ for (auto &s : codeStats) {
+ numTotalOrderedSections += s.numSections + s.numDuplicates;
+ totalOrderedSize += s.size + s.duplicateSize;
+ }
+ for (auto &s : dataStats) {
+ numTotalOrderedSections += s.numSections + s.numDuplicates;
+ totalOrderedSize += s.size + s.duplicateSize;
+ }
dbgs() << "Ordered " << numTotalOrderedSections << " sections ("
<< totalOrderedSize << " bytes) using balanced partitioning:\n";
dbgs() << " Functions for startup: " << numStartupSections << " ("
<< startupSize << " bytes)\n";
- dbgs() << " Functions for compression: " << numCodeCompressionSections
- << " (" << codeCompressionSize << " bytes)\n";
- dbgs() << " Duplicate functions: " << numDuplicateCodeSections << " ("
- << duplicateCodeSize << " bytes)\n";
- dbgs() << " Data for compression: " << numDataCompressionSections << " ("
- << dataCompressionSize << " bytes)\n";
- dbgs() << " Duplicate data: " << numDuplicateDataSections << " ("
- << duplicateDataSize << " bytes)\n";
+ dbgs() << " Code groups for compression: " << codeGroupIds.size() << "\n";
+ for (auto [i, groupId] : llvm::enumerate(codeGroupIds)) {
+ dbgs() << " Code group (" << groupId
+ << ") for compression: " << codeStats[i].numSections << " ("
+ << codeStats[i].size << " bytes)\n";
+ dbgs() << " Code group (" << groupId
+ << ") duplicates: " << codeStats[i].numDuplicates << " ("
+ << codeStats[i].duplicateSize << " bytes)\n";
+ }
+ dbgs() << " Data groups for compression: " << dataGroupIds.size() << "\n";
+ for (auto [i, groupId] : llvm::enumerate(dataGroupIds)) {
+ dbgs() << " Data group (" << groupId
+ << ") for compression: " << dataStats[i].numSections << " ("
+ << dataStats[i].size << " bytes)\n";
+ dbgs() << " Data group (" << groupId
+ << ") duplicates: " << dataStats[i].numDuplicates << " ("
+ << dataStats[i].duplicateSize << " bytes)\n";
+ }
if (!profilePath.empty()) {
// Evaluate this function order for startup
diff --git a/lld/test/ELF/bp-section-orderer-cold.s b/lld/test/ELF/bp-section-orderer-cold.s
new file mode 100644
index 0000000000000..3da22227cfc5a
--- /dev/null
+++ b/lld/test/ELF/bp-section-orderer-cold.s
@@ -0,0 +1,132 @@
+# REQUIRES: aarch64
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 a.s -o a.o
+# RUN: llvm-profdata merge a.proftext -o a.profdata
+
+## Compression sort: verify getCodeCompressionGroupId separates .text.unlikely
+## (group 1) and .text.split (group 2) from normal .text (group 0).
+# RUN: ld.lld a.o -o compr.out --bp-compression-sort=function --verbose-bp-section-orderer 2>&1 | FileCheck %s --check-prefix=COMPRESSION
+
+# COMPRESSION: Code groups for compression: 3
+# COMPRESSION: Code group (0) for compression: 4
+# COMPRESSION: Code group (1) for compression: 2
+# COMPRESSION: Code group (2) for compression: 1
+
+## Startup sort + compression sort: hot1 and cold1 are in the startup trace
+## and get ordered first.
+# RUN: ld.lld a.o -o startup-compr.out --irpgo-profile=a.profdata --bp-startup-sort=function --bp-compression-sort=function --verbose-bp-section-orderer 2>&1 | FileCheck %s --check-prefix=STARTUP-COMPR
+# RUN: llvm-nm -jn startup-compr.out | FileCheck %s --check-prefix=STARTUP-ORDER
+
+# STARTUP-ORDER-DAG: hot1
+# STARTUP-ORDER-DAG: cold1
+# STARTUP-ORDER: main
+# STARTUP-COMPR: Functions for startup: 2
+# STARTUP-COMPR: Code groups for compression: 3
+# STARTUP-COMPR: Code group (0) for compression: 3
+# STARTUP-COMPR: Code group (1) for compression: 1
+# STARTUP-COMPR: Code group (2) for compression: 1
+
+#--- a.proftext
+:ir
+:temporal_prof_traces
+# Num Traces
+1
+# Trace Stream Size:
+1
+# Weight
+1
+hot1, cold1
+
+hot1
+# Func Hash:
+1111
+# Num Counters:
+1
+# Counter Values:
+1
+
+cold1
+# Func Hash:
+2222
+# Num Counters:
+1
+# Counter Values:
+1
+
+#--- a.s
+ .section .text.main,"ax", at progbits
+ .globl main
+ .type main, at function
+main:
+ mov w0, wzr
+ ret
+.Lfunc_end_main:
+ .size main, .Lfunc_end_main-main
+
+ .section .text.unlikely.cold1,"ax", at progbits
+ .globl cold1
+ .type cold1, at function
+cold1:
+ add w0, w0, #10
+ add w1, w1, #11
+ bl main
+ ret
+.Lfunc_end_cold1:
+ .size cold1, .Lfunc_end_cold1-cold1
+
+ .section .text.hot1,"ax", at progbits
+ .globl hot1
+ .type hot1, at function
+hot1:
+ add w0, w0, #1
+ add w1, w1, #2
+ bl main
+ ret
+.Lfunc_end_hot1:
+ .size hot1, .Lfunc_end_hot1-hot1
+
+ .section .text.unlikely.cold2,"ax", at progbits
+ .globl cold2
+ .type cold2, at function
+cold2:
+ add w0, w0, #20
+ add w1, w1, #21
+ bl hot1
+ ret
+.Lfunc_end_cold2:
+ .size cold2, .Lfunc_end_cold2-cold2
+
+ .section .text.hot2,"ax", at progbits
+ .globl hot2
+ .type hot2, at function
+hot2:
+ add w0, w0, #2
+ add w1, w1, #3
+ bl hot1
+ ret
+.Lfunc_end_hot2:
+ .size hot2, .Lfunc_end_hot2-hot2
+
+ .section .text.split.cold_split1,"ax", at progbits
+ .globl cold_split1
+ .type cold_split1, at function
+cold_split1:
+ add w0, w0, #30
+ add w1, w1, #31
+ bl cold1
+ ret
+.Lfunc_end_cold_split1:
+ .size cold_split1, .Lfunc_end_cold_split1-cold_split1
+
+ .section .text.hot3,"ax", at progbits
+ .globl hot3
+ .type hot3, at function
+hot3:
+ add w0, w0, #3
+ add w1, w1, #4
+ bl cold1
+ ret
+.Lfunc_end_hot3:
+ .size hot3, .Lfunc_end_hot3-hot3
+
+ .section ".note.GNU-stack","", at progbits
More information about the llvm-commits
mailing list