[llvm] [llvm] Add API to get decompressed size of a zstd compressed buffer. (PR #107020)
Udit Kumar Agarwal via llvm-commits
llvm-commits at lists.llvm.org
Mon May 19 15:09:49 PDT 2025
https://github.com/uditagarwal97 updated https://github.com/llvm/llvm-project/pull/107020
>From 988da17b66a8861acdba31cd9feeae1651c9e03c Mon Sep 17 00:00:00 2001
From: "Agarwal, Udit" <udit.agarwal at intel.com>
Date: Mon, 2 Sep 2024 13:51:45 -0700
Subject: [PATCH 1/2] [llvm] Add API to get decompressed size of a zstd
compressed buffer.
---
llvm/include/llvm/Support/Compression.h | 2 ++
llvm/lib/Support/Compression.cpp | 18 ++++++++++++++++++
llvm/unittests/Support/CompressionTest.cpp | 9 ++++++++-
3 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h
index 2a8da9e96d356..da7b6f88d955b 100644
--- a/llvm/include/llvm/Support/Compression.h
+++ b/llvm/include/llvm/Support/Compression.h
@@ -71,6 +71,8 @@ Error decompress(ArrayRef<uint8_t> Input, uint8_t *Output,
Error decompress(ArrayRef<uint8_t> Input, SmallVectorImpl<uint8_t> &Output,
size_t UncompressedSize);
+// Get the size of the decompressed data.
+Error getDecompressedSize(ArrayRef<uint8_t> Input, size_t &UncompressedSize);
} // End of namespace zstd
enum class Format {
diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp
index badaf68ab59cd..ab012dfb5612b 100644
--- a/llvm/lib/Support/Compression.cpp
+++ b/llvm/lib/Support/Compression.cpp
@@ -224,6 +224,20 @@ Error zstd::decompress(ArrayRef<uint8_t> Input,
return E;
}
+Error zstd::getDecompressedSize(ArrayRef<uint8_t> Input,
+ size_t &UncompressedSize) {
+
+ unsigned long long Res = ZSTD_getFrameContentSize(Input.data(), Input.size());
+
+ // ZSTD_getFrameContentSize returns unsigned long long, but the size
+ // of uncompressed data should be bounded by size_t.
+ UncompressedSize = static_cast<size_t>(Res);
+
+ return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
+ inconvertibleErrorCode())
+ : Error::success();
+}
+
#else
bool zstd::isAvailable() { return false; }
void zstd::compress(ArrayRef<uint8_t> Input,
@@ -240,4 +254,8 @@ Error zstd::decompress(ArrayRef<uint8_t> Input,
size_t UncompressedSize) {
llvm_unreachable("zstd::decompress is unavailable");
}
+Error zstd::getDecompressedSize(ArrayRef<uint8_t> Input,
+ size_t &UncompressedSize) {
+ llvm_unreachable("zstd::getDecompressedSize is unavailable");
+}
#endif
diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp
index 5d326cafbe3a1..058be206d28b1 100644
--- a/llvm/unittests/Support/CompressionTest.cpp
+++ b/llvm/unittests/Support/CompressionTest.cpp
@@ -73,8 +73,15 @@ static void testZstdCompression(StringRef Input, int Level) {
SmallVector<uint8_t, 0> Uncompressed;
zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+ // Check that getDecompressedSize returns the size of the original buffer.
+ size_t DecompressedSize;
+ Error E = zstd::getDecompressedSize(Compressed, DecompressedSize);
+ EXPECT_FALSE(std::move(E));
+
+ EXPECT_EQ(DecompressedSize, Input.size());
+
// Check that uncompressed buffer is the same as original.
- Error E = zstd::decompress(Compressed, Uncompressed, Input.size());
+ E = zstd::decompress(Compressed, Uncompressed, Input.size());
EXPECT_FALSE(std::move(E));
EXPECT_EQ(Input, toStringRef(Uncompressed));
>From c9046d7b10ec896ab3b6bb9a7f6b5452f62feb4c Mon Sep 17 00:00:00 2001
From: "Agarwal, Udit" <udit.agarwal at intel.com>
Date: Mon, 19 May 2025 15:07:13 -0700
Subject: [PATCH 2/2] Use Expected<> instead
---
llvm/include/llvm/Support/Compression.h | 3 ++-
llvm/lib/Support/Compression.cpp | 13 +++++--------
llvm/unittests/Support/CompressionTest.cpp | 10 ++++------
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h
index d6eb812ed9092..64dadf96feb43 100644
--- a/llvm/include/llvm/Support/Compression.h
+++ b/llvm/include/llvm/Support/Compression.h
@@ -16,6 +16,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Error.h"
namespace llvm {
template <typename T> class SmallVectorImpl;
@@ -75,7 +76,7 @@ LLVM_ABI Error decompress(ArrayRef<uint8_t> Input,
size_t UncompressedSize);
// Get the size of the decompressed data.
-Error getDecompressedSize(ArrayRef<uint8_t> Input, size_t &UncompressedSize);
+Expected<uint64_t> getDecompressedSize(ArrayRef<uint8_t> Input);
} // End of namespace zstd
enum class Format {
diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp
index 4ea9fbf4b4f49..95bb1ec928c3f 100644
--- a/llvm/lib/Support/Compression.cpp
+++ b/llvm/lib/Support/Compression.cpp
@@ -225,18 +225,15 @@ Error zstd::decompress(ArrayRef<uint8_t> Input,
return E;
}
-Error zstd::getDecompressedSize(ArrayRef<uint8_t> Input,
- size_t &UncompressedSize) {
+Expected<uint64_t> zstd::getDecompressedSize(ArrayRef<uint8_t> Input) {
unsigned long long Res = ZSTD_getFrameContentSize(Input.data(), Input.size());
- // ZSTD_getFrameContentSize returns unsigned long long, but the size
- // of uncompressed data should be bounded by size_t.
- UncompressedSize = static_cast<size_t>(Res);
+ if (ZSTD_isError(Res))
+ return make_error<StringError>(ZSTD_getErrorName(Res),
+ inconvertibleErrorCode());
- return ZSTD_isError(Res) ? make_error<StringError>(ZSTD_getErrorName(Res),
- inconvertibleErrorCode())
- : Error::success();
+ return Res;
}
#else
diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp
index 058be206d28b1..4ad1d3ba4fec6 100644
--- a/llvm/unittests/Support/CompressionTest.cpp
+++ b/llvm/unittests/Support/CompressionTest.cpp
@@ -74,14 +74,12 @@ static void testZstdCompression(StringRef Input, int Level) {
zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
// Check that getDecompressedSize returns the size of the original buffer.
- size_t DecompressedSize;
- Error E = zstd::getDecompressedSize(Compressed, DecompressedSize);
- EXPECT_FALSE(std::move(E));
-
- EXPECT_EQ(DecompressedSize, Input.size());
+ Expected<uint64_t> DecompressedSize = zstd::getDecompressedSize(Compressed);
+ EXPECT_FALSE(!DecompressedSize);
+ EXPECT_EQ(DecompressedSize.get(), Input.size());
// Check that uncompressed buffer is the same as original.
- E = zstd::decompress(Compressed, Uncompressed, Input.size());
+ Error E = zstd::decompress(Compressed, Uncompressed, Input.size());
EXPECT_FALSE(std::move(E));
EXPECT_EQ(Input, toStringRef(Uncompressed));
More information about the llvm-commits
mailing list