[lld] [ELF] Adjust --compress-sections to support compression level (PR #90567)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Wed May 1 11:39:14 PDT 2024
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/90567
>From b5b923e67616fecbadb8a8c04f6dc7124f8f9d25 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Mon, 29 Apr 2024 23:41:50 -0700
Subject: [PATCH 1/3] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.5-bogner
---
lld/ELF/Config.h | 1 +
lld/ELF/Driver.cpp | 1 +
lld/ELF/Options.td | 3 +++
lld/ELF/OutputSections.cpp | 16 ++++++++--------
lld/docs/ld.lld.1 | 5 +++++
lld/test/ELF/compressed-debug-level.test | 19 ++++++++++---------
6 files changed, 28 insertions(+), 17 deletions(-)
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 33bfa42b0fcbf0..d619fe2bd5bf48 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -226,6 +226,7 @@ struct Config {
std::optional<llvm::DebugCompressionType> compressDebugSections;
llvm::SmallVector<std::pair<llvm::GlobPattern, llvm::DebugCompressionType>, 0>
compressSections;
+ int compressionLevel = 0;
bool cref;
llvm::SmallVector<std::pair<llvm::GlobPattern, uint64_t>, 0>
deadRelocInNonAlloc;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index a5b47f020f8726..3ef4a7a8bbfcad 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1233,6 +1233,7 @@ static void readConfigs(opt::InputArgList &args) {
config->compressDebugSections =
getCompressionType(arg->getValue(), "--compress-debug-sections");
}
+ config->compressionLevel = args::getInteger(args, OPT_compression_level, 0);
config->cref = args.hasArg(OPT_cref);
config->optimizeBBJumps =
args.hasFlag(OPT_optimize_bb_jumps, OPT_no_optimize_bb_jumps, false);
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 72eaf157a181cf..dba2e733af5d45 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -71,6 +71,9 @@ defm compress_sections: EEq<"compress-sections",
"Compress non-SHF_ALLOC output sections matching <section-glob>">,
MetaVarName<"<section-glob>=[none|zlib|zstd]">;
+defm compression_level: EEq<"compression-level", "Set the compression level">,
+ MetaVarName<"level">;
+
defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"<symbol>=<value>">;
defm optimize_bb_jumps: BB<"optimize-bb-jumps",
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 3e58ed4bda2d3c..51bd60d45d800f 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -372,13 +372,15 @@ template <class ELFT> void OutputSection::maybeCompress() {
auto shardsOut = std::make_unique<SmallVector<uint8_t, 0>[]>(numShards);
#if LLVM_ENABLE_ZSTD
- // Use ZSTD's streaming compression API which permits parallel workers working
- // on the stream. See http://facebook.github.io/zstd/zstd_manual.html
- // "Streaming compression - HowTo".
+ // Use ZSTD's streaming compression API. See
+ // http://facebook.github.io/zstd/zstd_manual.html "Streaming compression -
+ // HowTo".
if (ctype == DebugCompressionType::Zstd) {
parallelFor(0, numShards, [&](size_t i) {
SmallVector<uint8_t, 0> out;
ZSTD_CCtx *cctx = ZSTD_createCCtx();
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel,
+ config->compressionLevel);
ZSTD_inBuffer zib = {shardsIn[i].data(), shardsIn[i].size(), 0};
ZSTD_outBuffer zob = {nullptr, 0, 0};
size_t size;
@@ -406,12 +408,10 @@ template <class ELFT> void OutputSection::maybeCompress() {
#if LLVM_ENABLE_ZLIB
// We chose 1 (Z_BEST_SPEED) as the default compression level because it is
- // the fastest. If -O2 is given, we use level 6 to compress debug info more by
- // ~15%. We found that level 7 to 9 doesn't make much difference (~1% more
- // compression) while they take significant amount of time (~2x), so level 6
- // seems enough.
+ // the fastest.
if (ctype == DebugCompressionType::Zlib) {
- const int level = config->optimize >= 2 ? 6 : Z_BEST_SPEED;
+ const int level =
+ config->compressionLevel ? config->compressionLevel : Z_BEST_SPEED;
// Compress shards and compute Alder-32 checksums. Use Z_SYNC_FLUSH for all
// shards but the last to flush the output to a byte boundary to be
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 3861120915e8bc..90bc80329f06bd 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -172,6 +172,11 @@ This is like a generalized
Output cross reference table. If
.Fl Map
is specified, print to the map file.
+.It Fl -compression-level Ns = Ns level
+Set the compression level for
+.Fl -compress-debug-sections
+and
+.Fl -compress-sections.
.It Fl -debug-names
Generate a merged
.Li .debug_names
diff --git a/lld/test/ELF/compressed-debug-level.test b/lld/test/ELF/compressed-debug-level.test
index ee95f126799722..e911bf5ebcb030 100644
--- a/lld/test/ELF/compressed-debug-level.test
+++ b/lld/test/ELF/compressed-debug-level.test
@@ -2,22 +2,23 @@
# RUN: yaml2obj %s -o %t.o
+## LLD uses zlib compression of level 1 by default. Unlike previous versions,
+## -O does not change the level.
# RUN: ld.lld %t.o -o %t.default --compress-debug-sections=zlib
# RUN: llvm-readelf --sections %t.default | FileCheck -check-prefixes=HEADER,LEVEL1 %s
# RUN: ld.lld -O0 %t.o -o %t.O0 --compress-debug-sections=zlib
-# RUN: llvm-readelf --sections %t.O0 | FileCheck -check-prefixes=HEADER,LEVEL1 %s
# RUN: cmp %t.default %t.O0
-# RUN: ld.lld -O1 %t.o -o %t.O1 --compress-debug-sections=zlib
-# RUN: llvm-readelf --sections %t.O1 | FileCheck -check-prefixes=HEADER,LEVEL1 %s
-# RUN: cmp %t.default %t.O1
-
# RUN: ld.lld -O2 %t.o -o %t.O2 --compress-debug-sections=zlib
-# RUN: llvm-readelf --sections %t.O2 | FileCheck -check-prefixes=HEADER,LEVEL6 %s
-
-## LLD uses zlib compression of level 1 when -O0, -O1 and level 6 when -O2.
-## Here we check how -O flag affects the size of compressed sections produced.
+# RUN: cmp %t.default %t.O2
+
+## --compression-level specifies the level.
+# RUN: ld.lld --compression-level=6 %t.o -o %t.6 --compress-debug-sections=zlib
+# RUN: llvm-readelf --sections %t.6 | FileCheck -check-prefixes=HEADER,LEVEL6 %s
+## zlib uses the default (6) when an invalid level is specified.
+# RUN: ld.lld --compression-level=-1 %t.o -o %t.bad --compress-debug-sections=zlib
+# RUN: cmp %t.6 %t.bad
# HEADER: [Nr] Name Type Address Off Size
# LEVEL1: [ 1] .debug_info PROGBITS 00000000 000094 00001{{[bc]}}
>From ca98f5ae835c147e2eb58f0035e736d3b7e1bbb7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Tue, 30 Apr 2024 00:16:39 -0700
Subject: [PATCH 2/3] .
Created using spr 1.3.5-bogner
---
lld/docs/ReleaseNotes.rst | 3 +++
lld/test/ELF/compressed-debug-level.test | 4 ++++
2 files changed, 7 insertions(+)
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index a7ed49726fd99a..9039b7e9dd1302 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -29,6 +29,9 @@ ELF Improvements
* ``--compress-sections <section-glib>=[none|zlib|zstd]`` is added to compress
matched output sections without the ``SHF_ALLOC`` flag.
(`#84855 <https://github.com/llvm/llvm-project/pull/84855>`_)
+* ``--compression-level`` is added to set the compression level
+ for ``--compress-debug-sections=[zlib|zstd]`` and ``--compress-sections``.
+ (`#90567 <https://github.com/llvm/llvm-project/pull/90567>`_)
* ``GNU_PROPERTY_AARCH64_FEATURE_PAUTH`` notes, ``R_AARCH64_AUTH_ABS64`` and
``R_AARCH64_AUTH_RELATIVE`` relocations are now supported.
(`#72714 <https://github.com/llvm/llvm-project/pull/72714>`_)
diff --git a/lld/test/ELF/compressed-debug-level.test b/lld/test/ELF/compressed-debug-level.test
index e911bf5ebcb030..64fb8060acc2db 100644
--- a/lld/test/ELF/compressed-debug-level.test
+++ b/lld/test/ELF/compressed-debug-level.test
@@ -24,6 +24,10 @@
# LEVEL1: [ 1] .debug_info PROGBITS 00000000 000094 00001{{[bc]}}
# LEVEL6: [ 1] .debug_info PROGBITS 00000000 000094 00001a
+# RUN: not ld.lld --compression-level=a %t.o -o /dev/null --compress-debug-sections=zlib 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ERR: error: --compression-level=a: number expected, but got 'a'
+
## A little arbitrary debug section which has a different size after
## applying compression of level 1 and 6.
>From 88897aaa64cc31c193a8e13a15c61c1356bca4a7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Tue, 30 Apr 2024 12:53:34 -0700
Subject: [PATCH 3/3] improve test. change interaction with
--compress-debug-sections
Created using spr 1.3.5-bogner
---
lld/ELF/Options.td | 5 +++--
lld/ELF/OutputSections.cpp | 8 +++-----
lld/docs/ReleaseNotes.rst | 2 ++
lld/docs/ld.lld.1 | 7 ++-----
lld/test/ELF/compress-sections.s | 8 ++++----
lld/test/ELF/compressed-debug-level.test | 2 +-
6 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 72eaf157a181cf..f471ca822dfc18 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -68,8 +68,9 @@ defm compress_debug_sections:
MetaVarName<"[none,zlib,zstd]">;
defm compress_sections: EEq<"compress-sections",
- "Compress non-SHF_ALLOC output sections matching <section-glob>">,
- MetaVarName<"<section-glob>=[none|zlib|zstd]">;
+ "Compress non-SHF_ALLOC output sections matching <section-glob>. "
+ "If <level> is specified, use the specified compression level">,
+ MetaVarName<"<section-glob>={none,zlib,zstd}[:level]">;
defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"<symbol>=<value>">;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 92d424e81584df..9945251575a912 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -336,14 +336,12 @@ template <class ELFT> void OutputSection::maybeCompress() {
DebugCompressionType ctype = DebugCompressionType::None;
int level = 0; // default compression level
+ if (!(flags & SHF_ALLOC) && config->compressDebugSections &&
+ name.starts_with(".debug_") && size)
+ ctype = *config->compressDebugSections;
for (auto &[glob, t, l] : config->compressSections)
if (glob.match(name))
std::tie(ctype, level) = {t, l};
- if (!(flags & SHF_ALLOC) && config->compressDebugSections &&
- name.starts_with(".debug_") && size) {
- ctype = *config->compressDebugSections;
- level = 0;
- }
if (ctype == DebugCompressionType::None)
return;
if (flags & SHF_ALLOC) {
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 07d62d878e2c50..f8fdebfeaecf26 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -30,6 +30,8 @@ ELF Improvements
matched output sections without the ``SHF_ALLOC`` flag.
(`#84855 <https://github.com/llvm/llvm-project/pull/84855>`_)
(`#90567 <https://github.com/llvm/llvm-project/pull/90567>`_)
+* The default compression level for zlib is now independent of linker
+ optimization level (``Z_BEST_SPEED``).
* ``GNU_PROPERTY_AARCH64_FEATURE_PAUTH`` notes, ``R_AARCH64_AUTH_ABS64`` and
``R_AARCH64_AUTH_RELATIVE`` relocations are now supported.
(`#72714 <https://github.com/llvm/llvm-project/pull/72714>`_)
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 2af8a7982da199..76567f73481976 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -156,12 +156,9 @@ may be
No compression.
.It Cm zlib
The default compression level is 1 (fastest) as the debug info usually
-compresses well at that level. If you want to compress it more,
-you can specify
-.Fl O2
-to set the compression level to 6.
+compresses well at that level.
.It Cm zstd
-The compression level is 5.
+Use the default compression level in zstd.
.El
.Pp
.It Fl -compress-sections Ns = Ns Ar section-glob={none,zlib,zstd}[:level]
diff --git a/lld/test/ELF/compress-sections.s b/lld/test/ELF/compress-sections.s
index c9cfd5f564372e..a5107ee88a7dc9 100644
--- a/lld/test/ELF/compress-sections.s
+++ b/lld/test/ELF/compress-sections.s
@@ -16,7 +16,7 @@
# CHECK1: 0000000000000010 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc0) sym0
# CHECK1: 0000000000000008 0 NOTYPE LOCAL DEFAULT [[#]] (nonalloc1) sym1
-# RUN: ld.lld -pie a.o --compress-sections '*c0=zlib' --compress-sections .debug_str=zstd -o out2
+# RUN: ld.lld -pie a.o --compress-sections '*c0=zlib' --compress-sections .debug_str=zstd:3 -o out2
# RUN: llvm-readelf -SrsX -x nonalloc0 -x .debug_str out2 | FileCheck %s --check-prefix=CHECK2
# CHECK2: Name Type Address Off Size ES Flg Lk Inf Al
@@ -39,11 +39,11 @@
# CHECK2-NEXT: 02000000 00000000 38000000 00000000
# CHECK2-NEXT: 01000000 00000000 {{.*}}
-## --compress-debug-sections=none takes precedence.
-# RUN: ld.lld a.o --compress-debug-sections=none --compress-sections .debug_str=zstd -o out3
+## --compress-sections takes precedence.
+# RUN: ld.lld a.o --compress-sections .debug_str=zstd --compress-debug-sections=none -o out3
# RUN: llvm-readelf -S out3 | FileCheck %s --check-prefix=CHECK3
-# CHECK3: .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MS 0 0 1
+# CHECK3: .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MSC 0 0 1
# RUN: not ld.lld a.o --compress-sections '*0=zlib' 2>&1 | \
# RUN: FileCheck %s --check-prefix=ERR-ALLOC --implicit-check-not=error:
diff --git a/lld/test/ELF/compressed-debug-level.test b/lld/test/ELF/compressed-debug-level.test
index bca4124bc81145..004eeae308ca71 100644
--- a/lld/test/ELF/compressed-debug-level.test
+++ b/lld/test/ELF/compressed-debug-level.test
@@ -16,7 +16,7 @@
## --compression-level specifies the level.
# RUN: ld.lld %t.o -o %t.6 --compress-sections=.debug_info=zlib:6
# RUN: llvm-readelf --sections %t.6 | FileCheck -check-prefixes=HEADER,LEVEL6 %s
-## zlib uses the default (6) when an invalid level is specified.
+## If the specified level is invalid, zlib will use the default (6).
# RUN: ld.lld %t.o -o %t.bad --compress-sections=.debug_info=zlib:-1
# RUN: cmp %t.6 %t.bad
More information about the llvm-commits
mailing list