[lld] 449f2ca - [ELF] Add --compress-debug-sections=zstd
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 9 10:30:26 PDT 2022
Author: Fangrui Song
Date: 2022-09-09T10:30:18-07:00
New Revision: 449f2ca146dde0ef14c9837f71d4a0be92ca4843
URL: https://github.com/llvm/llvm-project/commit/449f2ca146dde0ef14c9837f71d4a0be92ca4843
DIFF: https://github.com/llvm/llvm-project/commit/449f2ca146dde0ef14c9837f71d4a0be92ca4843.diff
LOG: [ELF] Add --compress-debug-sections=zstd
`clang -gz=zstd a.o` passes this option to the linker. This option compresses output
debug sections with zstd and sets ch_type to ELFCOMPRESS_ZSTD. As of today, very
few DWARF consumers recognize ELFCOMPRESS_ZSTD.
Use the llvm::zstd::compress API with level llvm::zstd::DefaultCompression (5),
which we may tune after we have more experience with zstd output.
zstd has built-in parallel compression support (so we don't need to do D117853
for zlib), which is not leveraged yet.
Reviewed By: peter.smith
Differential Revision: https://reviews.llvm.org/D133548
Added:
Modified:
lld/ELF/Config.h
lld/ELF/Driver.cpp
lld/ELF/Options.td
lld/ELF/OutputSections.cpp
lld/docs/ReleaseNotes.rst
lld/docs/ld.lld.1
lld/test/ELF/compress-debug-sections-zstd.s
Removed:
################################################################################
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index accf1fd41713..1f8ff9362e91 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -19,6 +19,7 @@
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Support/CachePruning.h"
#include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/GlobPattern.h"
#include "llvm/Support/PrettyStackTrace.h"
@@ -164,7 +165,7 @@ struct Configuration {
bool callGraphProfileSort;
bool checkSections;
bool checkDynamicRelocs;
- bool compressDebugSections;
+ llvm::DebugCompressionType compressDebugSections;
bool cref;
llvm::SmallVector<std::pair<llvm::GlobPattern, uint64_t>, 0>
deadRelocInNonAlloc;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 650b3cce2461..369ef0294717 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -947,15 +947,21 @@ template <class ELFT> static void readCallGraphsFromObjectFiles() {
}
}
-static bool getCompressDebugSections(opt::InputArgList &args) {
+static DebugCompressionType getCompressDebugSections(opt::InputArgList &args) {
StringRef s = args.getLastArgValue(OPT_compress_debug_sections, "none");
- if (s == "none")
- return false;
- if (s != "zlib")
+ if (s == "zlib") {
+ if (!compression::zlib::isAvailable())
+ error("--compress-debug-sections: zlib is not available");
+ return DebugCompressionType::Zlib;
+ }
+ if (s == "zstd") {
+ if (!compression::zstd::isAvailable())
+ error("--compress-debug-sections: zstd is not available");
+ return DebugCompressionType::Zstd;
+ }
+ if (s != "none")
error("unknown --compress-debug-sections value: " + s);
- if (!compression::zlib::isAvailable())
- error("--compress-debug-sections: zlib is not available");
- return true;
+ return DebugCompressionType::None;
}
static StringRef getAliasSpelling(opt::Arg *arg) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index d9266e595887..72d5b6e2e663 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -60,7 +60,7 @@ defm check_sections: B<"check-sections",
defm compress_debug_sections:
Eq<"compress-debug-sections", "Compress DWARF debug sections">,
- MetaVarName<"[none,zlib]">;
+ MetaVarName<"[none,zlib,zstd]">;
defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"<symbol>=<value>">;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 48515b7be29a..517e8156330a 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -17,6 +17,7 @@
#include "lld/Common/Memory.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/Config/llvm-config.h" // LLVM_ENABLE_ZLIB
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Parallel.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/TimeProfiler.h"
@@ -320,18 +321,31 @@ static SmallVector<uint8_t, 0> deflateShard(ArrayRef<uint8_t> in, int level,
// Compress section contents if this section contains debug info.
template <class ELFT> void OutputSection::maybeCompress() {
-#if LLVM_ENABLE_ZLIB
using Elf_Chdr = typename ELFT::Chdr;
// Compress only DWARF debug sections.
- if (!config->compressDebugSections || (flags & SHF_ALLOC) ||
- !name.startswith(".debug_") || size == 0)
+ if (config->compressDebugSections == DebugCompressionType::None ||
+ (flags & SHF_ALLOC) || !name.startswith(".debug_") || size == 0)
return;
llvm::TimeTraceScope timeScope("Compress debug sections");
+ compressed.uncompressedSize = size;
+ auto buf = std::make_unique<uint8_t[]>(size);
+ if (config->compressDebugSections == DebugCompressionType::Zstd) {
+ {
+ parallel::TaskGroup tg;
+ writeTo<ELFT>(buf.get(), tg);
+ }
+ compressed.shards = std::make_unique<SmallVector<uint8_t, 0>[]>(1);
+ compression::zstd::compress(makeArrayRef(buf.get(), size),
+ compressed.shards[0]);
+ size = sizeof(Elf_Chdr) + compressed.shards[0].size();
+ flags |= SHF_COMPRESSED;
+ return;
+ }
+#if LLVM_ENABLE_ZLIB
// Write uncompressed data to a temporary zero-initialized buffer.
- auto buf = std::make_unique<uint8_t[]>(size);
{
parallel::TaskGroup tg;
writeTo<ELFT>(buf.get(), tg);
@@ -361,7 +375,6 @@ template <class ELFT> void OutputSection::maybeCompress() {
// Update section size and combine Alder-32 checksums.
uint32_t checksum = 1; // Initial Adler-32 value
- compressed.uncompressedSize = size;
size = sizeof(Elf_Chdr) + 2; // Elf_Chdir and zlib header
for (size_t i = 0; i != numShards; ++i) {
size += shardsOut[i].size();
@@ -400,10 +413,15 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
// just write it down.
if (compressed.shards) {
auto *chdr = reinterpret_cast<typename ELFT::Chdr *>(buf);
- chdr->ch_type = ELFCOMPRESS_ZLIB;
chdr->ch_size = compressed.uncompressedSize;
chdr->ch_addralign = alignment;
buf += sizeof(*chdr);
+ if (config->compressDebugSections == DebugCompressionType::Zstd) {
+ chdr->ch_type = ELFCOMPRESS_ZSTD;
+ memcpy(buf, compressed.shards[0].data(), compressed.shards[0].size());
+ return;
+ }
+ chdr->ch_type = ELFCOMPRESS_ZLIB;
// Compute shard offsets.
auto offsets = std::make_unique<size_t[]>(compressed.numShards);
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index af8b55578f81..2aad9b8644f3 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -28,6 +28,9 @@ ELF Improvements
* ``ELFCOMPRESS_ZSTD`` compressed input sections are now supported.
(`D129406 <https://reviews.llvm.org/D129406>`_)
+* ``--compress-debug-sections=zstd`` is now available to compress debug
+ sections with zstd (``ELFCOMPRESS_ZSTD``).
+ (`D133548 <https://reviews.llvm.org/D133548>`_)
Breaking changes
----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index b81eeb2232a2..7a4aa219187b 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -130,16 +130,22 @@ Alias for
.Fl -color-diagnostics Ns = Ns Cm auto .
.It Fl -compress-debug-sections Ns = Ns Ar value
Compress DWARF debug sections.
-.Ar value
+.Cm value
may be
-.Cm none
-or
-.Cm zlib .
+.Pp
+.Bl -tag -width 2n -compact
+.It Cm none
+No compression.
+.It Cm zlib
The default compression level is 1 (fastest) as the debug info usually
-compresses well at that level, but if you want to compress it more,
+compresses well at that level. If you want to compress it more,
you can specify
.Fl O2
to set the compression level to 6.
+.It Cm zstd
+The compression level is 5.
+.El
+.Pp
.It Fl -cref
Output cross reference table. If
.Fl Map
diff --git a/lld/test/ELF/compress-debug-sections-zstd.s b/lld/test/ELF/compress-debug-sections-zstd.s
index 59e4644c4e83..cf029c13c644 100644
--- a/lld/test/ELF/compress-debug-sections-zstd.s
+++ b/lld/test/ELF/compress-debug-sections-zstd.s
@@ -13,6 +13,13 @@
# CHECK-NEXT: 0x00000030 6e74006c 6f6e6720 756e7369 676e6564 nt.long unsigned
# CHECK-NEXT: 0x00000040 20696e74 00 int.
+# RUN: ld.lld %t.o -o %t.so -shared --compress-debug-sections=zstd
+# RUN: llvm-readelf -S %t.so | FileCheck %s --check-prefix=OUTPUT-SEC
+# RUN: llvm-objcopy --decompress-debug-sections %t.so
+# RUN: llvm-readelf -S -x .debug_str %t.so | FileCheck %s
+
+# OUTPUT-SEC: .debug_str PROGBITS [[#%x,]] [[#%x,]] [[#%x,]] 01 MSC 0 0 1
+
.section .debug_str,"MS", at progbits,1
.LASF2:
.string "short unsigned int"
More information about the llvm-commits
mailing list