[clang] 1c4c6f7 - [Offload][NFC] Deduplicate CompressedOffloadBundle into llvm::object (#206741)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 30 09:57:10 PDT 2026
Author: michaelselehov
Date: 2026-06-30T18:57:06+02:00
New Revision: 1c4c6f701ddd0645114f97b126654309a0d9dcb0
URL: https://github.com/llvm/llvm-project/commit/1c4c6f701ddd0645114f97b126654309a0d9dcb0
DIFF: https://github.com/llvm/llvm-project/commit/1c4c6f701ddd0645114f97b126654309a0d9dcb0.diff
LOG: [Offload][NFC] Deduplicate CompressedOffloadBundle into llvm::object (#206741)
The compressed offload bundle (CCOB) format had two byte-for-byte
identical implementations: one in clang/lib/Driver/OffloadBundler.cpp
and the canonical one in llvm/lib/Object/OffloadBundle.cpp (already used
by llvm-objdump, comgr and others).
Remove the Clang-side copy (class declaration, compress/decompress/
tryParse, the RawCompressedBundleHeader union, getHeaderSize, the
formatWithCommas helper and the private timer group) and route the
driver through llvm::object::CompressedOffloadBundle, which is already
linked via the Object component.
The only API difference is the verbose-reporting channel (bool Verbose
vs. raw_ostream *VerboseStream); the driver now passes &llvm::errs()
when verbose output is requested, preserving the previous behavior.
No functional change intended.
Co-authored-by: mselehov <mselehov at amd.com>
Added:
Modified:
clang/include/clang/Driver/OffloadBundler.h
clang/lib/Driver/OffloadBundler.cpp
Removed:
################################################################################
diff --git a/clang/include/clang/Driver/OffloadBundler.h b/clang/include/clang/Driver/OffloadBundler.h
index 637c9d146533f..96839240543af 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -92,42 +92,6 @@ struct OffloadTargetInfo {
std::string str() const;
};
-// CompressedOffloadBundle represents the format for the compressed offload
-// bundles.
-//
-// The format is as follows:
-// - Magic Number (4 bytes) - A constant "CCOB".
-// - Version (2 bytes)
-// - Compression Method (2 bytes) - Uses the values from
-// llvm::compression::Format.
-// - Total file size (4 bytes in V2, 8 bytes in V3).
-// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
-// - Truncated MD5 Hash (8 bytes).
-// - Compressed Data (variable length).
-class CompressedOffloadBundle {
-private:
- static inline const llvm::StringRef MagicNumber = "CCOB";
-
-public:
- struct CompressedBundleHeader {
- unsigned Version;
- llvm::compression::Format CompressionFormat;
- std::optional<size_t> FileSize;
- size_t UncompressedFileSize;
- uint64_t Hash;
-
- static llvm::Expected<CompressedBundleHeader> tryParse(llvm::StringRef);
- };
-
- static inline const uint16_t DefaultVersion = 3;
-
- static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
- compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input,
- uint16_t Version, bool Verbose = false);
- static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
- decompress(const llvm::MemoryBuffer &Input, bool Verbose = false);
-};
-
/// Check whether the bundle id is in the following format:
/// <kind>-<triple>[-<target id>[:target features]]
/// <triple> := <arch>-<vendor>-<os>-<env>
diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp
index 9d6f32c4a4e8f..4c7d13212f968 100644
--- a/clang/lib/Driver/OffloadBundler.cpp
+++ b/clang/lib/Driver/OffloadBundler.cpp
@@ -28,6 +28,7 @@
#include "llvm/Object/ArchiveWriter.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/OffloadBundle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Compression.h"
@@ -65,18 +66,6 @@ using namespace llvm;
using namespace llvm::object;
using namespace clang;
-namespace {
-struct CreateClangOffloadBundlerTimerGroup {
- static void *call() {
- return new TimerGroup("Clang Offload Bundler Timer Group",
- "Timer group for clang offload bundler");
- }
-};
-} // namespace
-static llvm::ManagedStatic<llvm::TimerGroup,
- CreateClangOffloadBundlerTimerGroup>
- ClangOffloadBundlerTimerGroup;
-
/// Magic string that marks the existence of offloading data.
#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
@@ -1013,336 +1002,6 @@ OffloadBundlerConfig::OffloadBundlerConfig()
}
}
-// Utility function to format numbers with commas
-static std::string formatWithCommas(unsigned long long Value) {
- std::string Num = std::to_string(Value);
- int InsertPosition = Num.length() - 3;
- while (InsertPosition > 0) {
- Num.insert(InsertPosition, ",");
- InsertPosition -= 3;
- }
- return Num;
-}
-
-llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
-CompressedOffloadBundle::compress(llvm::compression::Params P,
- const llvm::MemoryBuffer &Input,
- uint16_t Version, bool Verbose) {
- if (!llvm::compression::zstd::isAvailable() &&
- !llvm::compression::zlib::isAvailable())
- return createStringError(llvm::inconvertibleErrorCode(),
- "Compression not supported");
- llvm::Timer HashTimer("Hash Calculation Timer", "Hash calculation time",
- *ClangOffloadBundlerTimerGroup);
- if (Verbose)
- HashTimer.startTimer();
- llvm::MD5 Hash;
- llvm::MD5::MD5Result Result;
- Hash.update(Input.getBuffer());
- Hash.final(Result);
- uint64_t TruncatedHash = Result.low();
- if (Verbose)
- HashTimer.stopTimer();
-
- SmallVector<uint8_t, 0> CompressedBuffer;
- auto BufferUint8 = llvm::ArrayRef<uint8_t>(
- reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),
- Input.getBuffer().size());
- llvm::Timer CompressTimer("Compression Timer", "Compression time",
- *ClangOffloadBundlerTimerGroup);
- if (Verbose)
- CompressTimer.startTimer();
- llvm::compression::compress(P, BufferUint8, CompressedBuffer);
- if (Verbose)
- CompressTimer.stopTimer();
-
- uint16_t CompressionMethod = static_cast<uint16_t>(P.format);
-
- // Store sizes in 64-bit variables first
- uint64_t UncompressedSize64 = Input.getBuffer().size();
- uint64_t TotalFileSize64;
-
- // Calculate total file size based on version
- if (Version == 2) {
- // For V2, ensure the sizes don't exceed 32-bit limit
- if (UncompressedSize64 > std::numeric_limits<uint32_t>::max())
- return createStringError(llvm::inconvertibleErrorCode(),
- "Uncompressed size exceeds version 2 limit");
- if ((MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) +
- sizeof(CompressionMethod) + sizeof(uint32_t) + sizeof(TruncatedHash) +
- CompressedBuffer.size()) > std::numeric_limits<uint32_t>::max())
- return createStringError(llvm::inconvertibleErrorCode(),
- "Total file size exceeds version 2 limit");
-
- TotalFileSize64 = MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) +
- sizeof(CompressionMethod) + sizeof(uint32_t) +
- sizeof(TruncatedHash) + CompressedBuffer.size();
- } else { // Version 3
- TotalFileSize64 = MagicNumber.size() + sizeof(uint64_t) + sizeof(Version) +
- sizeof(CompressionMethod) + sizeof(uint64_t) +
- sizeof(TruncatedHash) + CompressedBuffer.size();
- }
-
- SmallVector<char, 0> FinalBuffer;
- llvm::raw_svector_ostream OS(FinalBuffer);
- OS << MagicNumber;
- OS.write(reinterpret_cast<const char *>(&Version), sizeof(Version));
- OS.write(reinterpret_cast<const char *>(&CompressionMethod),
- sizeof(CompressionMethod));
-
- // Write size fields according to version
- if (Version == 2) {
- uint32_t TotalFileSize32 = static_cast<uint32_t>(TotalFileSize64);
- uint32_t UncompressedSize32 = static_cast<uint32_t>(UncompressedSize64);
- OS.write(reinterpret_cast<const char *>(&TotalFileSize32),
- sizeof(TotalFileSize32));
- OS.write(reinterpret_cast<const char *>(&UncompressedSize32),
- sizeof(UncompressedSize32));
- } else { // Version 3
- OS.write(reinterpret_cast<const char *>(&TotalFileSize64),
- sizeof(TotalFileSize64));
- OS.write(reinterpret_cast<const char *>(&UncompressedSize64),
- sizeof(UncompressedSize64));
- }
-
- OS.write(reinterpret_cast<const char *>(&TruncatedHash),
- sizeof(TruncatedHash));
- OS.write(reinterpret_cast<const char *>(CompressedBuffer.data()),
- CompressedBuffer.size());
-
- if (Verbose) {
- auto MethodUsed =
- P.format == llvm::compression::Format::Zstd ? "zstd" : "zlib";
- double CompressionRate =
- static_cast<double>(UncompressedSize64) / CompressedBuffer.size();
- double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime();
- double CompressionSpeedMBs =
- (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds;
- llvm::errs() << "Compressed bundle format version: " << Version << "\n"
- << "Total file size (including headers): "
- << formatWithCommas(TotalFileSize64) << " bytes\n"
- << "Compression method used: " << MethodUsed << "\n"
- << "Compression level: " << P.level << "\n"
- << "Binary size before compression: "
- << formatWithCommas(UncompressedSize64) << " bytes\n"
- << "Binary size after compression: "
- << formatWithCommas(CompressedBuffer.size()) << " bytes\n"
- << "Compression rate: "
- << llvm::format("%.2lf", CompressionRate) << "\n"
- << "Compression ratio: "
- << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
- << "Compression speed: "
- << llvm::format("%.2lf MB/s", CompressionSpeedMBs) << "\n"
- << "Truncated MD5 hash: "
- << llvm::format_hex(TruncatedHash, 16) << "\n";
- }
-
- return llvm::MemoryBuffer::getMemBufferCopy(
- llvm::StringRef(FinalBuffer.data(), FinalBuffer.size()));
-}
-
-// Use packed structs to avoid padding, such that the structs map the serialized
-// format.
-LLVM_PACKED_START
-union RawCompressedBundleHeader {
- struct CommonFields {
- uint32_t Magic;
- uint16_t Version;
- uint16_t Method;
- };
-
- struct V1Header {
- CommonFields Common;
- uint32_t UncompressedFileSize;
- uint64_t Hash;
- };
-
- struct V2Header {
- CommonFields Common;
- uint32_t FileSize;
- uint32_t UncompressedFileSize;
- uint64_t Hash;
- };
-
- struct V3Header {
- CommonFields Common;
- uint64_t FileSize;
- uint64_t UncompressedFileSize;
- uint64_t Hash;
- };
-
- CommonFields Common;
- V1Header V1;
- V2Header V2;
- V3Header V3;
-};
-LLVM_PACKED_END
-
-// Helper method to get header size based on version
-static size_t getHeaderSize(uint16_t Version) {
- switch (Version) {
- case 1:
- return sizeof(RawCompressedBundleHeader::V1Header);
- case 2:
- return sizeof(RawCompressedBundleHeader::V2Header);
- case 3:
- return sizeof(RawCompressedBundleHeader::V3Header);
- default:
- llvm_unreachable("Unsupported version");
- }
-}
-
-Expected<CompressedOffloadBundle::CompressedBundleHeader>
-CompressedOffloadBundle::CompressedBundleHeader::tryParse(StringRef Blob) {
- assert(Blob.size() >= sizeof(RawCompressedBundleHeader::CommonFields));
- assert(llvm::identify_magic(Blob) ==
- llvm::file_magic::offload_bundle_compressed);
-
- RawCompressedBundleHeader Header;
- memcpy(&Header, Blob.data(), std::min(Blob.size(), sizeof(Header)));
-
- CompressedBundleHeader Normalized;
- Normalized.Version = Header.Common.Version;
-
- size_t RequiredSize = getHeaderSize(Normalized.Version);
- if (Blob.size() < RequiredSize)
- return createStringError(inconvertibleErrorCode(),
- "Compressed bundle header size too small");
-
- switch (Normalized.Version) {
- case 1:
- Normalized.UncompressedFileSize = Header.V1.UncompressedFileSize;
- Normalized.Hash = Header.V1.Hash;
- break;
- case 2:
- Normalized.FileSize = Header.V2.FileSize;
- Normalized.UncompressedFileSize = Header.V2.UncompressedFileSize;
- Normalized.Hash = Header.V2.Hash;
- break;
- case 3:
- Normalized.FileSize = Header.V3.FileSize;
- Normalized.UncompressedFileSize = Header.V3.UncompressedFileSize;
- Normalized.Hash = Header.V3.Hash;
- break;
- default:
- return createStringError(inconvertibleErrorCode(),
- "Unknown compressed bundle version");
- }
-
- // Determine compression format
- switch (Header.Common.Method) {
- case static_cast<uint16_t>(compression::Format::Zlib):
- case static_cast<uint16_t>(compression::Format::Zstd):
- Normalized.CompressionFormat =
- static_cast<compression::Format>(Header.Common.Method);
- break;
- default:
- return createStringError(inconvertibleErrorCode(),
- "Unknown compressing method");
- }
-
- return Normalized;
-}
-
-llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
-CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input,
- bool Verbose) {
- StringRef Blob = Input.getBuffer();
-
- // Check minimum header size (using V1 as it's the smallest)
- if (Blob.size() < sizeof(RawCompressedBundleHeader::CommonFields))
- return llvm::MemoryBuffer::getMemBufferCopy(Blob);
-
- if (llvm::identify_magic(Blob) !=
- llvm::file_magic::offload_bundle_compressed) {
- if (Verbose)
- llvm::errs() << "Uncompressed bundle.\n";
- return llvm::MemoryBuffer::getMemBufferCopy(Blob);
- }
-
- Expected<CompressedBundleHeader> HeaderOrErr =
- CompressedBundleHeader::tryParse(Blob);
- if (!HeaderOrErr)
- return HeaderOrErr.takeError();
-
- const CompressedBundleHeader &Normalized = *HeaderOrErr;
- unsigned ThisVersion = Normalized.Version;
- size_t HeaderSize = getHeaderSize(ThisVersion);
-
- llvm::compression::Format CompressionFormat = Normalized.CompressionFormat;
-
- size_t TotalFileSize = Normalized.FileSize.value_or(0);
- size_t UncompressedSize = Normalized.UncompressedFileSize;
- auto StoredHash = Normalized.Hash;
-
- llvm::Timer DecompressTimer("Decompression Timer", "Decompression time",
- *ClangOffloadBundlerTimerGroup);
- if (Verbose)
- DecompressTimer.startTimer();
-
- SmallVector<uint8_t, 0> DecompressedData;
- StringRef CompressedData =
- Blob.substr(HeaderSize, TotalFileSize - HeaderSize);
- if (llvm::Error DecompressionError = llvm::compression::decompress(
- CompressionFormat, llvm::arrayRefFromStringRef(CompressedData),
- DecompressedData, UncompressedSize))
- return createStringError(inconvertibleErrorCode(),
- "Could not decompress embedded file contents: " +
- llvm::toString(std::move(DecompressionError)));
-
- if (Verbose) {
- DecompressTimer.stopTimer();
-
- double DecompressionTimeSeconds =
- DecompressTimer.getTotalTime().getWallTime();
-
- // Recalculate MD5 hash for integrity check
- llvm::Timer HashRecalcTimer("Hash Recalculation Timer",
- "Hash recalculation time",
- *ClangOffloadBundlerTimerGroup);
- HashRecalcTimer.startTimer();
- llvm::MD5 Hash;
- llvm::MD5::MD5Result Result;
- Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData));
- Hash.final(Result);
- uint64_t RecalculatedHash = Result.low();
- HashRecalcTimer.stopTimer();
- bool HashMatch = (StoredHash == RecalculatedHash);
-
- double CompressionRate =
- static_cast<double>(UncompressedSize) / CompressedData.size();
- double DecompressionSpeedMBs =
- (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
-
- llvm::errs() << "Compressed bundle format version: " << ThisVersion << "\n";
- if (ThisVersion >= 2)
- llvm::errs() << "Total file size (from header): "
- << formatWithCommas(TotalFileSize) << " bytes\n";
- llvm::errs() << "Decompression method: "
- << (CompressionFormat == llvm::compression::Format::Zlib
- ? "zlib"
- : "zstd")
- << "\n"
- << "Size before decompression: "
- << formatWithCommas(CompressedData.size()) << " bytes\n"
- << "Size after decompression: "
- << formatWithCommas(UncompressedSize) << " bytes\n"
- << "Compression rate: "
- << llvm::format("%.2lf", CompressionRate) << "\n"
- << "Compression ratio: "
- << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n"
- << "Decompression speed: "
- << llvm::format("%.2lf MB/s", DecompressionSpeedMBs) << "\n"
- << "Stored hash: " << llvm::format_hex(StoredHash, 16) << "\n"
- << "Recalculated hash: "
- << llvm::format_hex(RecalculatedHash, 16) << "\n"
- << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";
- }
-
- return llvm::MemoryBuffer::getMemBufferCopy(
- llvm::toStringRef(DecompressedData));
-}
-
// List bundle IDs. Return true if an error was found.
Error OffloadBundler::ListBundleIDsInFile(
StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) {
@@ -1380,7 +1039,8 @@ Error OffloadBundler::ListBundleIDsInFile(
// Decompress the input if necessary.
Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
- CompressedOffloadBundle::decompress(**CodeOrErr, BundlerConfig.Verbose);
+ CompressedOffloadBundle::decompress(
+ **CodeOrErr, BundlerConfig.Verbose ? &llvm::errs() : nullptr);
if (!DecompressedBufferOrErr)
return createStringError(
inconvertibleErrorCode(),
@@ -1562,7 +1222,7 @@ Error OffloadBundler::BundleFiles() {
{BundlerConfig.CompressionFormat, BundlerConfig.CompressionLevel,
/*zstdEnableLdm=*/true},
*BufferMemory, BundlerConfig.CompressedBundleVersion,
- BundlerConfig.Verbose);
+ BundlerConfig.Verbose ? &llvm::errs() : nullptr);
if (auto Error = CompressionResult.takeError())
return Error;
@@ -1633,7 +1293,8 @@ Error OffloadBundler::UnbundleFiles() {
// Decompress the blob if necessary.
Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
- CompressedOffloadBundle::decompress(**BlobOrErr, BundlerConfig.Verbose);
+ CompressedOffloadBundle::decompress(
+ **BlobOrErr, BundlerConfig.Verbose ? &llvm::errs() : nullptr);
if (!DecompressedBufferOrErr)
return createStringError(
inconvertibleErrorCode(),
@@ -1928,8 +1589,9 @@ Error OffloadBundler::UnbundleArchive() {
// Decompress the buffer if necessary.
Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
- CompressedOffloadBundle::decompress(*TempCodeObjectBuffer,
- BundlerConfig.Verbose);
+ CompressedOffloadBundle::decompress(
+ *TempCodeObjectBuffer,
+ BundlerConfig.Verbose ? &llvm::errs() : nullptr);
if (!DecompressedBufferOrErr)
return createStringError(
inconvertibleErrorCode(),
More information about the cfe-commits
mailing list