[llvm] [LLVM] Add zstd compressed frame magic to LLVM magic (PR #222773)
Joseph Huber via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 06:15:34 PDT 2026
https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/222773
>From ed191213a44899051cb776064ea17335da69bfd2 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Thu, 10 Sep 2026 13:53:35 -0500
Subject: [PATCH 1/2] [LLVM] Add zstd compressed frame magic to LLVM magic
Summary:
ZSTD uses magic to indentify compressed frames in the bitstream. Add
this as a recognized file magic type so that we can identify compressed
streams that LLVM creates.
---
llvm/include/llvm/BinaryFormat/Magic.h | 1 +
llvm/lib/BinaryFormat/Magic.cpp | 5 ++++
llvm/lib/Object/Binary.cpp | 1 +
llvm/lib/Object/ObjectFile.cpp | 1 +
llvm/unittests/BinaryFormat/CMakeLists.txt | 1 +
llvm/unittests/BinaryFormat/TestFileMagic.cpp | 29 +++++++++++++++++--
6 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/BinaryFormat/Magic.h b/llvm/include/llvm/BinaryFormat/Magic.h
index 11922d843af9e..ecc40212dbd2f 100644
--- a/llvm/include/llvm/BinaryFormat/Magic.h
+++ b/llvm/include/llvm/BinaryFormat/Magic.h
@@ -60,6 +60,7 @@ struct file_magic {
offload_bundle, ///< Clang offload bundle file
offload_bundle_compressed, ///< Compressed clang offload bundle file
spirv_object, ///< A binary SPIR-V file
+ zstd, ///< Zstandard compressed frame
};
bool is_object() const { return V != unknown; }
diff --git a/llvm/lib/BinaryFormat/Magic.cpp b/llvm/lib/BinaryFormat/Magic.cpp
index 0786c4b9889fb..ce5d2eb248806 100644
--- a/llvm/lib/BinaryFormat/Magic.cpp
+++ b/llvm/lib/BinaryFormat/Magic.cpp
@@ -270,6 +270,11 @@ file_magic llvm::identify_magic(StringRef Magic) {
return file_magic::coff_object;
break;
+ case 0x28: // Zstandard compressed frame magic 0xFD2FB528, little-endian.
+ if (startswith(Magic, "\x28\xb5\x2f\xfd"))
+ return file_magic::zstd;
+ break;
+
case '_': {
const char OBMagic[] = "__CLANG_OFFLOAD_BUNDLE__";
if (Magic.size() >= sizeof(OBMagic) && startswith(Magic, OBMagic))
diff --git a/llvm/lib/Object/Binary.cpp b/llvm/lib/Object/Binary.cpp
index 30414257fa90b..5cb10d7ae25c2 100644
--- a/llvm/lib/Object/Binary.cpp
+++ b/llvm/lib/Object/Binary.cpp
@@ -91,6 +91,7 @@ Expected<std::unique_ptr<Binary>> object::createBinary(MemoryBufferRef Buffer,
case file_magic::offload_bundle:
case file_magic::offload_bundle_compressed:
case file_magic::spirv_object:
+ case file_magic::zstd:
// Unrecognized object file format.
return errorCodeToError(object_error::invalid_file_type);
case file_magic::offload_binary: {
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index c4b4cdff6de93..f045fda97683f 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -168,6 +168,7 @@ ObjectFile::createObjectFile(MemoryBufferRef Object, file_magic Type,
case file_magic::offload_bundle:
case file_magic::offload_bundle_compressed:
case file_magic::spirv_object:
+ case file_magic::zstd:
return errorCodeToError(object_error::invalid_file_type);
case file_magic::tapi_file:
return errorCodeToError(object_error::invalid_file_type);
diff --git a/llvm/unittests/BinaryFormat/CMakeLists.txt b/llvm/unittests/BinaryFormat/CMakeLists.txt
index eac5977a2c1c3..b89d38e47f602 100644
--- a/llvm/unittests/BinaryFormat/CMakeLists.txt
+++ b/llvm/unittests/BinaryFormat/CMakeLists.txt
@@ -1,5 +1,6 @@
set(LLVM_LINK_COMPONENTS
BinaryFormat
+ Support
TargetParser
)
diff --git a/llvm/unittests/BinaryFormat/TestFileMagic.cpp b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
index 7d235d7c84842..42fffc1868e15 100644
--- a/llvm/unittests/BinaryFormat/TestFileMagic.cpp
+++ b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
@@ -6,9 +6,13 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
@@ -92,6 +96,7 @@ const char tapi_file[] = "--- !tapi-tbd-v1\n";
const char tapi_file_tbd_v1[] = "---\narchs: [";
const char spirv_object_le[] = "\x03\x02\x23\x07";
const char spirv_object_be[] = "\x07\x23\x02\x03";
+const char zstd[] = "\x28\xb5\x2f\xfd";
TEST_F(MagicTest, Magic) {
struct type {
@@ -126,9 +131,10 @@ TEST_F(MagicTest, Magic) {
{"macho_type_0x10001", macho_type_0x10001, sizeof(macho_type_0x10001),
file_magic::macho_object},
{"spirv_object_le", spirv_object_le, sizeof(spirv_object_le),
- file_magic ::spirv_object},
+ file_magic::spirv_object},
{"spirv_object_be", spirv_object_be, sizeof(spirv_object_be),
- file_magic ::spirv_object},
+ file_magic::spirv_object},
+ DEFINE(zstd),
DEFINE(windows_resource),
DEFINE(pdb),
{"ms_dos_stub_broken", ms_dos_stub_broken, sizeof(ms_dos_stub_broken),
@@ -154,3 +160,22 @@ TEST_F(MagicTest, Magic) {
ASSERT_NO_ERROR(fs::remove(Twine(file_pathname)));
}
}
+
+TEST_F(MagicTest, ZstdCompressedBlob) {
+ if (!compression::zstd::isAvailable())
+ GTEST_SKIP() << "zstd is not available";
+
+ static const int Levels[] = {compression::zstd::NoCompression,
+ compression::zstd::BestSpeedCompression,
+ compression::zstd::DefaultCompression,
+ compression::zstd::BestSizeCompression};
+
+ StringRef Input("hello, world!");
+ for (int Level : Levels) {
+ SmallVector<uint8_t, 0> Compressed;
+ compression::zstd::compress(arrayRefFromStringRef(Input), Compressed,
+ Level);
+ EXPECT_EQ(file_magic::zstd, identify_magic(toStringRef(Compressed)))
+ << "level=" << Level;
+ }
+}
>From ef4593da4f7b59c4ae024d6acb4ffa6eba253e40 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Fri, 11 Sep 2026 08:09:56 -0500
Subject: [PATCH 2/2] [LLVM] Add zlib compressed data magic to LLVM magic
RFC 1950 CMF/FLG with a 32K window is four well-defined headers, which
is what LLVM's compress2 emits. Recognize those the same way as zstd so
callers can name either stream without defaulting unknown bytes to zlib.
---
llvm/include/llvm/BinaryFormat/Magic.h | 1 +
llvm/lib/BinaryFormat/Magic.cpp | 10 +++++
llvm/lib/Object/Binary.cpp | 1 +
llvm/lib/Object/ObjectFile.cpp | 1 +
llvm/unittests/BinaryFormat/TestFileMagic.cpp | 42 +++++++++++++++----
5 files changed, 48 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/BinaryFormat/Magic.h b/llvm/include/llvm/BinaryFormat/Magic.h
index ecc40212dbd2f..764602286994e 100644
--- a/llvm/include/llvm/BinaryFormat/Magic.h
+++ b/llvm/include/llvm/BinaryFormat/Magic.h
@@ -61,6 +61,7 @@ struct file_magic {
offload_bundle_compressed, ///< Compressed clang offload bundle file
spirv_object, ///< A binary SPIR-V file
zstd, ///< Zstandard compressed frame
+ zlib, ///< zlib compressed data
};
bool is_object() const { return V != unknown; }
diff --git a/llvm/lib/BinaryFormat/Magic.cpp b/llvm/lib/BinaryFormat/Magic.cpp
index ce5d2eb248806..f5f53896595d8 100644
--- a/llvm/lib/BinaryFormat/Magic.cpp
+++ b/llvm/lib/BinaryFormat/Magic.cpp
@@ -275,6 +275,16 @@ file_magic llvm::identify_magic(StringRef Magic) {
return file_magic::zstd;
break;
+ case 0x78: {
+ // RFC 1950 zlib wrapper. LLVM's compress2 uses a 32K window (CMF 0x78).
+ // FCHECK makes CMF*256+FLG a multiple of 31. FDICT is unused in LLVM which
+ // exposes four identifiable two-byte headers.
+ unsigned char Flg = Magic[1];
+ if ((Flg & 0x20) == 0 && (0x7800u + Flg) % 31 == 0)
+ return file_magic::zlib;
+ break;
+ }
+
case '_': {
const char OBMagic[] = "__CLANG_OFFLOAD_BUNDLE__";
if (Magic.size() >= sizeof(OBMagic) && startswith(Magic, OBMagic))
diff --git a/llvm/lib/Object/Binary.cpp b/llvm/lib/Object/Binary.cpp
index 5cb10d7ae25c2..c7f3614d4f2a2 100644
--- a/llvm/lib/Object/Binary.cpp
+++ b/llvm/lib/Object/Binary.cpp
@@ -92,6 +92,7 @@ Expected<std::unique_ptr<Binary>> object::createBinary(MemoryBufferRef Buffer,
case file_magic::offload_bundle_compressed:
case file_magic::spirv_object:
case file_magic::zstd:
+ case file_magic::zlib:
// Unrecognized object file format.
return errorCodeToError(object_error::invalid_file_type);
case file_magic::offload_binary: {
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index f045fda97683f..044204013806e 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -169,6 +169,7 @@ ObjectFile::createObjectFile(MemoryBufferRef Object, file_magic Type,
case file_magic::offload_bundle_compressed:
case file_magic::spirv_object:
case file_magic::zstd:
+ case file_magic::zlib:
return errorCodeToError(object_error::invalid_file_type);
case file_magic::tapi_file:
return errorCodeToError(object_error::invalid_file_type);
diff --git a/llvm/unittests/BinaryFormat/TestFileMagic.cpp b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
index 42fffc1868e15..a78fc17426361 100644
--- a/llvm/unittests/BinaryFormat/TestFileMagic.cpp
+++ b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
@@ -12,6 +12,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Config/llvm-config.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
@@ -97,6 +98,7 @@ const char tapi_file_tbd_v1[] = "---\narchs: [";
const char spirv_object_le[] = "\x03\x02\x23\x07";
const char spirv_object_be[] = "\x07\x23\x02\x03";
const char zstd[] = "\x28\xb5\x2f\xfd";
+const char zlib[] = "\x78\x9c\x00\x00";
TEST_F(MagicTest, Magic) {
struct type {
@@ -135,6 +137,7 @@ TEST_F(MagicTest, Magic) {
{"spirv_object_be", spirv_object_be, sizeof(spirv_object_be),
file_magic::spirv_object},
DEFINE(zstd),
+ DEFINE(zlib),
DEFINE(windows_resource),
DEFINE(pdb),
{"ms_dos_stub_broken", ms_dos_stub_broken, sizeof(ms_dos_stub_broken),
@@ -161,14 +164,12 @@ TEST_F(MagicTest, Magic) {
}
}
+#if LLVM_ENABLE_ZSTD
TEST_F(MagicTest, ZstdCompressedBlob) {
- if (!compression::zstd::isAvailable())
- GTEST_SKIP() << "zstd is not available";
-
- static const int Levels[] = {compression::zstd::NoCompression,
- compression::zstd::BestSpeedCompression,
- compression::zstd::DefaultCompression,
- compression::zstd::BestSizeCompression};
+ constexpr int Levels[] = {compression::zstd::NoCompression,
+ compression::zstd::BestSpeedCompression,
+ compression::zstd::DefaultCompression,
+ compression::zstd::BestSizeCompression};
StringRef Input("hello, world!");
for (int Level : Levels) {
@@ -179,3 +180,30 @@ TEST_F(MagicTest, ZstdCompressedBlob) {
<< "level=" << Level;
}
}
+#endif
+
+#if LLVM_ENABLE_ZLIB
+TEST_F(MagicTest, ZlibCompressedBlob) {
+ constexpr int Levels[] = {compression::zlib::NoCompression,
+ compression::zlib::BestSpeedCompression,
+ compression::zlib::DefaultCompression,
+ compression::zlib::BestSizeCompression};
+
+ StringRef Input("hello, world!");
+ for (int Level : Levels) {
+ SmallVector<uint8_t, 0> Compressed;
+ compression::zlib::compress(arrayRefFromStringRef(Input), Compressed,
+ Level);
+ EXPECT_EQ(file_magic::zlib, identify_magic(toStringRef(Compressed)))
+ << "level=" << Level;
+ }
+
+ // The four CMF/FLG pairs file(1) treats as zlib (FDICT clear, FCHECK valid).
+ EXPECT_EQ(file_magic::zlib, identify_magic(StringRef("\x78\x01\x00\x00", 4)));
+ EXPECT_EQ(file_magic::zlib, identify_magic(StringRef("\x78\x5e\x00\x00", 4)));
+ EXPECT_EQ(file_magic::zlib, identify_magic(StringRef("\x78\x9c\x00\x00", 4)));
+ EXPECT_EQ(file_magic::zlib, identify_magic(StringRef("\x78\xda\x00\x00", 4)));
+ EXPECT_EQ(file_magic::unknown,
+ identify_magic(StringRef("\x78\x00\x00\x00", 4)));
+}
+#endif
More information about the llvm-commits
mailing list