[llvm] 7759daa - [LLVM] Infer compression format from zlib and zstd headers (#222773)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 05:48:06 PDT 2026
Author: Joseph Huber
Date: 2026-09-29T07:47:57-05:00
New Revision: 7759daa2e9a57a4fe6445531b405217096f416bd
URL: https://github.com/llvm/llvm-project/commit/7759daa2e9a57a4fe6445531b405217096f416bd
DIFF: https://github.com/llvm/llvm-project/commit/7759daa2e9a57a4fe6445531b405217096f416bd.diff
LOG: [LLVM] Infer compression format from zlib and zstd headers (#222773)
Summary:
This PR adds two routines that allow users to decompress bitstreams
without explicitly specifying the format. This is inferred from the
bytes
of the object, returning an error if it could not be identified. The
Zstd
standard exposes proper magic bytes, while Zlib needs to be inferred
through the two header bytes.
Because this is a raw bytestream, there is a chance that someone
could get bytes that pass this check, giving no 'reason', but since
the decompression routine requires the expected size there is no
chance it will falsely succeed.
Added:
Modified:
clang/lib/Serialization/ASTReader.cpp
llvm/include/llvm/Support/Compression.h
llvm/lib/Support/Compression.cpp
llvm/unittests/Support/CompressionTest.cpp
Removed:
################################################################################
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 86d419ce6e7ab..120df65ce2919 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1927,19 +1927,9 @@ bool ASTReader::ReadSLocEntry(int ID) {
unsigned RecCode = MaybeRecCode.get();
if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) {
- // Inspect the first byte to
diff erentiate zlib (\x78) and zstd
- // (little-endian 0xFD2FB528).
- const llvm::compression::Format F =
- Blob.size() > 0 && Blob.data()[0] == 0x78
- ? llvm::compression::Format::Zlib
- : llvm::compression::Format::Zstd;
- if (const char *Reason = llvm::compression::getReasonIfUnsupported(F)) {
- Error(Reason);
- return nullptr;
- }
SmallVector<uint8_t, 0> Decompressed;
if (llvm::Error E = llvm::compression::decompress(
- F, llvm::arrayRefFromStringRef(Blob), Decompressed, Record[0])) {
+ llvm::arrayRefFromStringRef(Blob), Decompressed, Record[0])) {
Error("could not decompress embedded file contents: " +
llvm::toString(std::move(E)));
return nullptr;
diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h
index 3cfee87dceab8..b7307f8264b1f 100644
--- a/llvm/include/llvm/Support/Compression.h
+++ b/llvm/include/llvm/Support/Compression.h
@@ -130,6 +130,9 @@ struct Params {
// return a string literal describing the reason.
LLVM_ABI const char *getReasonIfUnsupported(Format F);
+// Return nullptr if LLVM can decompress Input, otherwise a string literal.
+LLVM_ABI const char *getReasonIfUnsupported(ArrayRef<uint8_t> Input);
+
// Compress Input with the specified format P.Format. If Level is -1, use
// *::DefaultCompression for the format.
LLVM_ABI void compress(Params P, ArrayRef<uint8_t> Input,
@@ -144,6 +147,10 @@ LLVM_ABI Error decompress(Format F, ArrayRef<uint8_t> Input,
LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
SmallVectorImpl<uint8_t> &Output,
size_t UncompressedSize);
+// Infer the compression format from the input.
+LLVM_ABI Error decompress(ArrayRef<uint8_t> Input,
+ SmallVectorImpl<uint8_t> &Output,
+ size_t UncompressedSize);
} // End of namespace compression
diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp
index 80dfa45d17650..5d2dee317e29a 100644
--- a/llvm/lib/Support/Compression.cpp
+++ b/llvm/lib/Support/Compression.cpp
@@ -19,6 +19,7 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include <limits>
+#include <optional>
#if LLVM_ENABLE_ZLIB
#include <zlib.h>
#endif
@@ -32,6 +33,37 @@
using namespace llvm;
using namespace llvm::compression;
+// RFC 1950 section 2.2 zlib wrapper. Two-byte header CMF then FLG:
+// CMF: CM (bits 0-3) must be 8 (deflate). CINFO (bits 4-7) is
+// log2(windowSize)-8 and must be <= 7.
+// FLG: FCHECK (bits 0-4) is chosen so CMF*256+FLG is a multiple of 31;
+// FDICT (bit 5) marks a preset dictionary; FLEVEL (bits 6-7) is a
+// compressor hint. This only identifies the wrapper.
+static bool isZlibHeader(ArrayRef<uint8_t> Input) {
+ if (Input.size() < 2)
+ return false;
+ unsigned CMF = Input[0];
+ unsigned FLG = Input[1];
+ if ((CMF & 0x0f) != 8 || (CMF >> 4) > 7)
+ return false;
+ return (CMF * 256 + FLG) % 31 == 0;
+}
+
+// RFC 8878 section 3.1.1: Zstandard frame magic 0xFD2FB528, little-endian.
+static bool isZstdMagic(ArrayRef<uint8_t> Input) {
+ static constexpr uint8_t Magic[] = {0x28, 0xb5, 0x2f, 0xfd};
+ return Input.take_front(4) == ArrayRef(Magic);
+}
+
+// Check zstd first: 0x28 is a valid zlib CMF (CINFO=2, 1KiB window).
+static std::optional<Format> identifyFormat(ArrayRef<uint8_t> Input) {
+ if (isZstdMagic(Input))
+ return Format::Zstd;
+ if (isZlibHeader(Input))
+ return Format::Zlib;
+ return std::nullopt;
+}
+
const char *compression::getReasonIfUnsupported(compression::Format F) {
switch (F) {
case compression::Format::Zlib:
@@ -48,6 +80,12 @@ const char *compression::getReasonIfUnsupported(compression::Format F) {
llvm_unreachable("");
}
+const char *compression::getReasonIfUnsupported(ArrayRef<uint8_t> Input) {
+ if (std::optional<Format> F = identifyFormat(Input))
+ return getReasonIfUnsupported(*F);
+ return "unknown compression format";
+}
+
void compression::compress(Params P, ArrayRef<uint8_t> Input,
SmallVectorImpl<uint8_t> &Output) {
switch (P.format) {
@@ -89,6 +127,16 @@ Error compression::decompress(DebugCompressionType T, ArrayRef<uint8_t> Input,
return decompress(formatFor(T), Input, Output, UncompressedSize);
}
+Error compression::decompress(ArrayRef<uint8_t> Input,
+ SmallVectorImpl<uint8_t> &Output,
+ size_t UncompressedSize) {
+ std::optional<Format> F = identifyFormat(Input);
+ if (const char *Reason =
+ F ? getReasonIfUnsupported(*F) : "unknown compression format")
+ return createStringError(Reason);
+ return decompress(*F, Input, Output, UncompressedSize);
+}
+
#if LLVM_ENABLE_ZLIB
static StringRef convertZlibCodeToString(int Code) {
diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp
index df6b4540251b4..dd1210ba160b9 100644
--- a/llvm/unittests/Support/CompressionTest.cpp
+++ b/llvm/unittests/Support/CompressionTest.cpp
@@ -40,6 +40,11 @@ static void testZlibCompression(StringRef Input, int Level) {
EXPECT_FALSE(std::move(E));
EXPECT_EQ(Input, toStringRef(Uncompressed));
+ // decompress infers zlib from the RFC 1950 header.
+ E = compression::decompress(Compressed, Uncompressed, Input.size());
+ EXPECT_FALSE(std::move(E));
+ EXPECT_EQ(Input, toStringRef(Uncompressed));
+
if (Input.size() > 0) {
// Decompression fails if expected length is too short.
E = zlib::decompress(Compressed, Uncompressed, Input.size() - 1);
@@ -85,6 +90,11 @@ static void testZstdCompression(StringRef Input, int Level) {
EXPECT_FALSE(std::move(E));
EXPECT_EQ(Input, toStringRef(Uncompressed));
+ // decompress infers Zstd from the frame magic.
+ E = compression::decompress(Compressed, Uncompressed, Input.size());
+ EXPECT_FALSE(std::move(E));
+ EXPECT_EQ(Input, toStringRef(Uncompressed));
+
if (Input.size() > 0) {
// Decompression fails if expected length is too short.
E = zstd::decompress(Compressed, Uncompressed, Input.size() - 1);
@@ -195,4 +205,36 @@ TEST(CompressionTest, XzDecompressErrors) {
xzDecompressError(FlipByte(XzStreamHeaderSize + BlockHeaderSize)));
}
#endif
+
+TEST(CompressionTest, IdentifyHeaders) {
+ EXPECT_STREQ("unknown compression format",
+ getReasonIfUnsupported(ArrayRef<uint8_t>()));
+ uint8_t Truncated[] = {0x78};
+ EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(Truncated));
+
+ // RFC 1950 headers LLVM's compress2 does not emit.
+ uint8_t SmallWindow[] = {0x28, 0x15}; // CINFO=2, FCHECK valid
+ EXPECT_EQ(getReasonIfUnsupported(Format::Zlib),
+ getReasonIfUnsupported(ArrayRef<uint8_t>(SmallWindow)));
+ uint8_t WithDict[] = {0x78, 0x20}; // FDICT set, FCHECK valid
+ EXPECT_EQ(getReasonIfUnsupported(Format::Zlib),
+ getReasonIfUnsupported(ArrayRef<uint8_t>(WithDict)));
+
+ uint8_t BadFCheck[] = {0x78, 0x00};
+ EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadFCheck));
+ uint8_t BadCINFO[] = {0x88, 0x01};
+ EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadCINFO));
+ uint8_t BadCM[] = {0x79, 0x9c};
+ EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadCM));
+
+ uint8_t ZstdMagic[] = {0x28, 0xb5, 0x2f, 0xfd};
+ EXPECT_EQ(getReasonIfUnsupported(Format::Zstd),
+ getReasonIfUnsupported(ArrayRef<uint8_t>(ZstdMagic)));
+
+ uint8_t Unknown[] = {0x00, 0x01, 0x02, 0x03};
+ EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(Unknown));
+ SmallVector<uint8_t, 0> Out;
+ Error E = compression::decompress(ArrayRef<uint8_t>(Unknown), Out, 0);
+ EXPECT_EQ("unknown compression format", toString(std::move(E)));
+}
} // namespace
More information about the llvm-commits
mailing list