[llvm] c26dc29 - [llvm-objcopy] Support --{,de}compress-debug-sections for zstd
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 28 10:45:57 PDT 2022
Author: Fangrui Song
Date: 2022-07-28T10:45:53-07:00
New Revision: c26dc2904b95b3685d883e760e84046ea6c33d7f
URL: https://github.com/llvm/llvm-project/commit/c26dc2904b95b3685d883e760e84046ea6c33d7f
DIFF: https://github.com/llvm/llvm-project/commit/c26dc2904b95b3685d883e760e84046ea6c33d7f.diff
LOG: [llvm-objcopy] Support --{,de}compress-debug-sections for zstd
Also, add ELFCOMPRESS_ZSTD (2) from the approved generic-abi proposal:
https://groups.google.com/g/generic-abi/c/satyPkuMisk
("Add new ch_type value: ELFCOMPRESS_ZSTD")
Link: https://discourse.llvm.org/t/rfc-zstandard-as-a-second-compression-method-to-llvm/63399
("[RFC] Zstandard as a second compression method to LLVM")
Differential Revision: https://reviews.llvm.org/D130458
Added:
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd-err.test
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test
llvm/test/tools/llvm-objcopy/ELF/decompress-debug-sections-err.test
Modified:
llvm/docs/CommandGuide/llvm-objcopy.rst
llvm/include/llvm/BinaryFormat/ELF.h
llvm/include/llvm/MC/MCTargetOptions.h
llvm/lib/ObjCopy/ELF/ELFObject.cpp
llvm/lib/ObjCopy/ELF/ELFObject.h
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
llvm/tools/llvm-objcopy/ObjcopyOpts.td
llvm/utils/lit/lit/llvm/config.py
Removed:
################################################################################
diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst
index 01bb43ecbf48b..8894a7fea6bb7 100644
--- a/llvm/docs/CommandGuide/llvm-objcopy.rst
+++ b/llvm/docs/CommandGuide/llvm-objcopy.rst
@@ -299,7 +299,7 @@ them.
.. option:: --compress-debug-sections [<format>]
Compress DWARF debug sections in the output, using the specified format.
- Supported formats are ``zlib``. Use ``zlib`` if ``<format>`` is omitted.
+ Supported formats are ``zlib`` and ``zstd``. Use ``zlib`` if ``<format>`` is omitted.
.. option:: --decompress-debug-sections
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index a0bb50db8c544..721aa7d2e193b 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1798,6 +1798,7 @@ struct Elf64_Nhdr {
// Legal values for ch_type field of compressed section header.
enum {
ELFCOMPRESS_ZLIB = 1, // ZLIB/DEFLATE algorithm.
+ ELFCOMPRESS_ZSTD = 2, // Zstandard algorithm
ELFCOMPRESS_LOOS = 0x60000000, // Start of OS-specific.
ELFCOMPRESS_HIOS = 0x6fffffff, // End of OS-specific.
ELFCOMPRESS_LOPROC = 0x70000000, // Start of processor-specific.
diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h
index ae305564a3536..e52633c6f8a0c 100644
--- a/llvm/include/llvm/MC/MCTargetOptions.h
+++ b/llvm/include/llvm/MC/MCTargetOptions.h
@@ -27,7 +27,8 @@ enum class ExceptionHandling {
enum class DebugCompressionType {
None, ///< No compression
- Z, ///< zlib style complession
+ Z, ///< zlib
+ Zstd, ///< Zstandard
};
enum class EmitDwarfUnwindType {
diff --git a/llvm/lib/ObjCopy/ELF/ELFObject.cpp b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
index b127e1b43b8e5..3d31b7d9c554e 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObject.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
@@ -438,14 +438,32 @@ template <class ELFT>
Error ELFSectionWriter<ELFT>::visit(const DecompressedSection &Sec) {
ArrayRef<uint8_t> Compressed =
Sec.OriginalData.slice(sizeof(Elf_Chdr_Impl<ELFT>));
- SmallVector<uint8_t, 128> DecompressedContent;
- if (Error Err = compression::zlib::uncompress(Compressed, DecompressedContent,
+ SmallVector<uint8_t, 128> Decompressed;
+ auto Report = [&](Error Err) {
+ return createStringError(errc::invalid_argument,
+ "failed to decompress section '" + Sec.Name +
+ "': " + toString(std::move(Err)));
+ };
+ switch (Sec.ChType) {
+ case ELFCOMPRESS_ZLIB:
+ if (Error E = compression::zlib::uncompress(Compressed, Decompressed,
+ static_cast<size_t>(Sec.Size)))
+ return Report(std::move(E));
+ break;
+ case ELFCOMPRESS_ZSTD:
+ if (Error E = compression::zstd::uncompress(Compressed, Decompressed,
static_cast<size_t>(Sec.Size)))
+ return Report(std::move(E));
+ break;
+ default:
return createStringError(errc::invalid_argument,
- "'" + Sec.Name + "': " + toString(std::move(Err)));
+ "--decompress-debug-sections: ch_type (" +
+ Twine(Sec.ChType) + ") of section '" +
+ Sec.Name + "' is unsupported");
+ }
uint8_t *Buf = reinterpret_cast<uint8_t *>(Out.getBufferStart()) + Sec.Offset;
- std::copy(DecompressedContent.begin(), DecompressedContent.end(), Buf);
+ std::copy(Decompressed.begin(), Decompressed.end(), Buf);
return Error::success();
}
@@ -498,6 +516,9 @@ Error ELFSectionWriter<ELFT>::visit(const CompressedSection &Sec) {
case DebugCompressionType::Z:
Chdr.ch_type = ELF::ELFCOMPRESS_ZLIB;
break;
+ case DebugCompressionType::Zstd:
+ Chdr.ch_type = ELF::ELFCOMPRESS_ZSTD;
+ break;
}
Chdr.ch_size = Sec.DecompressedSize;
Chdr.ch_addralign = Sec.DecompressedAlign;
@@ -512,9 +533,17 @@ CompressedSection::CompressedSection(const SectionBase &Sec,
DebugCompressionType CompressionType)
: SectionBase(Sec), CompressionType(CompressionType),
DecompressedSize(Sec.OriginalData.size()), DecompressedAlign(Sec.Align) {
- compression::zlib::compress(OriginalData, CompressedData);
+ switch (CompressionType) {
+ case DebugCompressionType::Z:
+ compression::zlib::compress(OriginalData, CompressedData);
+ break;
+ case DebugCompressionType::Zstd:
+ compression::zstd::compress(OriginalData, CompressedData);
+ break;
+ case DebugCompressionType::None:
+ llvm_unreachable("should not use CompressedSection");
+ }
- assert(CompressionType != DebugCompressionType::None);
Flags |= ELF::SHF_COMPRESSED;
size_t ChdrSize =
std::max(std::max(sizeof(object::Elf_Chdr_Impl<object::ELF64LE>),
@@ -526,9 +555,9 @@ CompressedSection::CompressedSection(const SectionBase &Sec,
}
CompressedSection::CompressedSection(ArrayRef<uint8_t> CompressedData,
- uint64_t DecompressedSize,
+ uint32_t ChType, uint64_t DecompressedSize,
uint64_t DecompressedAlign)
- : CompressionType(DebugCompressionType::None),
+ : ChType(ChType), CompressionType(DebugCompressionType::None),
DecompressedSize(DecompressedSize), DecompressedAlign(DecompressedAlign) {
OriginalData = CompressedData;
}
@@ -1706,8 +1735,8 @@ Expected<SectionBase &> ELFBuilder<ELFT>::makeSection(const Elf_Shdr &Shdr) {
if (!(Shdr.sh_flags & ELF::SHF_COMPRESSED))
return Obj.addSection<Section>(*Data);
auto *Chdr = reinterpret_cast<const Elf_Chdr_Impl<ELFT> *>(Data->data());
- return Obj.addSection<CompressedSection>(
- CompressedSection(*Data, Chdr->ch_size, Chdr->ch_addralign));
+ return Obj.addSection<CompressedSection>(CompressedSection(
+ *Data, Chdr->ch_type, Chdr->ch_size, Chdr->ch_addralign));
}
}
}
diff --git a/llvm/lib/ObjCopy/ELF/ELFObject.h b/llvm/lib/ObjCopy/ELF/ELFObject.h
index 2c3ea3a5f6d6a..7fb4e05405e73 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObject.h
+++ b/llvm/lib/ObjCopy/ELF/ELFObject.h
@@ -536,6 +536,7 @@ class OwnedDataSection : public SectionBase {
class CompressedSection : public SectionBase {
MAKE_SEC_WRITER_FRIEND
+ uint32_t ChType = 0;
DebugCompressionType CompressionType;
uint64_t DecompressedSize;
uint64_t DecompressedAlign;
@@ -544,11 +545,12 @@ class CompressedSection : public SectionBase {
public:
CompressedSection(const SectionBase &Sec,
DebugCompressionType CompressionType);
- CompressedSection(ArrayRef<uint8_t> CompressedData, uint64_t DecompressedSize,
- uint64_t DecompressedAlign);
+ CompressedSection(ArrayRef<uint8_t> CompressedData, uint32_t ChType,
+ uint64_t DecompressedSize, uint64_t DecompressedAlign);
uint64_t getDecompressedSize() const { return DecompressedSize; }
uint64_t getDecompressedAlign() const { return DecompressedAlign; }
+ uint64_t getChType() const { return ChType; }
Error accept(SectionVisitor &Visitor) const override;
Error accept(MutableSectionVisitor &Visitor) override;
@@ -562,8 +564,9 @@ class DecompressedSection : public SectionBase {
MAKE_SEC_WRITER_FRIEND
public:
+ uint32_t ChType;
explicit DecompressedSection(const CompressedSection &Sec)
- : SectionBase(Sec) {
+ : SectionBase(Sec), ChType(Sec.getChType()) {
Size = Sec.getDecompressedSize();
Align = Sec.getDecompressedAlign();
Flags = OriginalFlags = (Flags & ~ELF::SHF_COMPRESSED);
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test
index 02f0bb97f2c0e..c2481b7386d69 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test
@@ -2,12 +2,15 @@
# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
# RUN: llvm-objcopy --compress-debug-sections %t.o %t-compressed.o
-# RUN: llvm-readobj -S %t-compressed.o | FileCheck %s
+# RUN: llvm-readobj -S --sd %t-compressed.o | FileCheck %s
# CHECK: Name: .debug_foo
# CHECK-NEXT: Type: SHT_PROGBITS
# CHECK-NEXT: Flags [
# CHECK-NEXT: SHF_COMPRESSED
# CHECK-NEXT: ]
+# CHECK: SectionData (
+## ch_type = ELFCOMPRESS_ZLIB (1)
+# CHECK-NEXT: 0000: 01000000 {{.*}}
# CHECK-NOT: Name: .debug_foo
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
index 80844f794400f..e1ebeed8d4fcb 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
@@ -28,3 +28,21 @@
## --compress-debug-sections does not update a compressed section.
# RUN: llvm-objcopy --compress-debug-sections=zlib %t-zlib.o %t-zlib-zlib.o
# RUN: cmp %t-zlib.o %t-zlib-zlib.o
+
+# RUN: yaml2obj %s -o %t-corrupted
+# RUN: not llvm-objcopy --decompress-debug-sections %t-corrupted /dev/null 2>&1 | FileCheck %s -DFILE=%t-corrupted --check-prefix=ERR
+
+# ERR: error: '[[FILE]]': failed to decompress section '.debug_info': zlib error: Z_DATA_ERROR
+
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2LSB
+ Type: ET_REL
+ Machine: EM_X86_64
+Sections:
+ - Type: SHT_PROGBITS
+ Name: .debug_info
+ Flags: [ SHF_COMPRESSED ]
+ AddressAlign: 8
+ Content: "010000000000000004000000000000000100000000000000ffffffff"
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd-err.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd-err.test
new file mode 100644
index 0000000000000..1bc8ef22608f0
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd-err.test
@@ -0,0 +1,5 @@
+# UNSUPPORTED: zstd
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t
+# RUN: not llvm-objcopy --compress-debug-sections=zstd %t /dev/null 2>&1 | FileCheck %s
+
+# CHECK: error: LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot compress
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test
new file mode 100644
index 0000000000000..f71450c95bad2
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test
@@ -0,0 +1,51 @@
+# REQUIRES: zstd
+## Test --compress-debug-sections=zstd and decompression.
+
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t && llvm-objcopy %t
+# RUN: llvm-objcopy --compress-debug-sections=zstd %t %t-zstd
+# RUN: llvm-objcopy --decompress-debug-sections %t-zstd %t-de
+# RUN: cmp %t %t-de
+
+# RUN: llvm-readelf -S -r -x .debug_foo %t-zstd | FileCheck %s --check-prefixes=CHECK,COMPRESSED
+# RUN: llvm-readelf -S -r -x .debug_foo %t-de | FileCheck %s --check-prefixes=CHECK,DECOMPRESSED
+
+# CHECK: Name Type Address Off Size ES Flg Lk Inf Al
+# COMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 {{.*}} 00 C 0 0 8
+# COMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0
+# DECOMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 000008 00 0 0 0
+# DECOMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0
+
+## Relocations do not change.
+# CHECK: Relocation section '.rela.debug_foo' at offset {{.*}} contains 2 entries:
+# CHECK-NEXT: Offset
+# CHECK-NEXT: 0000000000000001 000000010000000a R_X86_64_32 0000000000000000 .debug_foo + 0
+# CHECK-NEXT: 0000000000000002 000000020000000a R_X86_64_32 0000000000000000 .notdebug_foo + 0
+
+# COMPRESSED: Hex dump of section '.debug_foo':
+## ch_type == ELFCOMPRESS_ZSTD (2)
+# COMPRESSED-NEXT: 0x00000000 02000000 00000000 08000000 00000000
+# COMPRESSED-NEXT: 0x00000010 00000000 00000000 {{.*}}
+
+## --compress-debug-sections does not update a compressed section. Its compression
+## type does not change.
+# RUN: llvm-objcopy --compress-debug-sections=zstd %t-zstd %t-zstd-zstd
+# RUN: cmp %t-zstd %t-zstd-zstd
+# RUN: %if zlib %{ llvm-objcopy --compress-debug-sections=zlib %t-zstd %t-zstd-zlib && cmp %t-zstd %t-zstd-zlib %}
+
+# RUN: yaml2obj %s -o %t-corrupted
+# RUN: not llvm-objcopy --decompress-debug-sections %t-corrupted /dev/null 2>&1 | FileCheck %s -DFILE=%t-corrupted --check-prefix=ERR
+
+# ERR: error: '[[FILE]]': failed to decompress section '.debug_info': Src size is incorrect
+
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2LSB
+ Type: ET_REL
+ Machine: EM_X86_64
+Sections:
+ - Type: SHT_PROGBITS
+ Name: .debug_info
+ Flags: [ SHF_COMPRESSED ]
+ AddressAlign: 8
+ Content: "020000000000000004000000000000000100000000000000ffffffff"
diff --git a/llvm/test/tools/llvm-objcopy/ELF/decompress-debug-sections-err.test b/llvm/test/tools/llvm-objcopy/ELF/decompress-debug-sections-err.test
new file mode 100644
index 0000000000000..e3d8396564908
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/decompress-debug-sections-err.test
@@ -0,0 +1,24 @@
+# REQUIRES: zlib
+# RUN: yaml2obj %s -o %t
+# RUN: not llvm-objcopy --decompress-debug-sections %t /dev/null 2>&1 | FileCheck %s -DFILE=%t
+
+# CHECK: error: '[[FILE]]': --decompress-debug-sections: ch_type (3) of section '.debug_info' is unsupported
+# CHECK-EMPTY:
+
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2LSB
+ Type: ET_REL
+ Machine: EM_X86_64
+Sections:
+ - Name: .debug_info
+ Type: SHT_PROGBITS
+ Flags: [ SHF_COMPRESSED ]
+ Content: 030000000000000004000000000000000000000000000000789c6360
+ AddressAlign: 8
+ - Name: .debug_str
+ Type: SHT_PROGBITS
+ Flags: [ SHF_COMPRESSED ]
+ Content: 030000000000000004000000000000000000000000000000789c6360
+ AddressAlign: 8
diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
index 7db1e79f3e49a..7d4e651f973a7 100644
--- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -722,16 +722,27 @@ objcopy::parseObjcopyOptions(ArrayRef<const char *> RawArgsArr,
if (const auto *A = InputArgs.getLastArg(OBJCOPY_compress_debug_sections)) {
Config.CompressionType = StringSwitch<DebugCompressionType>(A->getValue())
.Case("zlib", DebugCompressionType::Z)
+ .Case("zstd", DebugCompressionType::Zstd)
.Default(DebugCompressionType::None);
- if (Config.CompressionType == DebugCompressionType::None)
+ switch (Config.CompressionType) {
+ case DebugCompressionType::None:
return createStringError(
errc::invalid_argument,
"invalid or unsupported --compress-debug-sections format: %s",
A->getValue());
- if (!compression::zlib::isAvailable())
- return createStringError(
- errc::invalid_argument,
- "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
+ case DebugCompressionType::Z:
+ if (!compression::zlib::isAvailable())
+ return createStringError(
+ errc::invalid_argument,
+ "LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot compress");
+ break;
+ case DebugCompressionType::Zstd:
+ if (!compression::zstd::isAvailable())
+ return createStringError(
+ errc::invalid_argument,
+ "LLVM was not compiled with LLVM_ENABLE_ZSTD: cannot compress");
+ break;
+ }
}
Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
diff --git a/llvm/tools/llvm-objcopy/ObjcopyOpts.td b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
index d3713b5ec3c32..0fddd443a4cc7 100644
--- a/llvm/tools/llvm-objcopy/ObjcopyOpts.td
+++ b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
@@ -33,7 +33,7 @@ def compress_debug_sections
: Joined<["--"], "compress-debug-sections=">,
MetaVarName<"format">,
HelpText<"Compress DWARF debug sections using specified format. Supported "
- "formats: zlib">;
+ "formats: zlib, zstd. Select zlib if <format> is omitted">;
def : Flag<["--"], "compress-debug-sections">, Alias<compress_debug_sections>,
AliasArgs<["zlib"]>;
def decompress_debug_sections : Flag<["--"], "decompress-debug-sections">,
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index b653161281464..19e9004f24087 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -108,6 +108,9 @@ def __init__(self, lit_config, config):
have_zlib = getattr(config, 'have_zlib', None)
if have_zlib:
features.add('zlib')
+ have_zstd = getattr(config, 'have_zstd', None)
+ if have_zstd:
+ features.add('zstd')
# Check if we should run long running tests.
long_tests = lit_config.params.get('run_long_tests', None)
More information about the llvm-commits
mailing list