[llvm] Extend LLVM Offloading API for binary fatbin Bundles (PR #114833)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 4 09:23:17 PST 2024
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {clang-format}-->
:warning: C/C++ code formatter, clang-format found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
git-clang-format --diff 70af40ba74cf62fdaa3ae1d7db972c138655049f 71c9c5cb43c750ce35136b183d28a8a138d6f6e5 --extensions h,cpp -- llvm/include/llvm/Object/OffloadBinary.h llvm/lib/Object/OffloadBinary.cpp
``````````
</details>
<details>
<summary>
View the diff from clang-format here.
</summary>
``````````diff
diff --git a/llvm/include/llvm/Object/OffloadBinary.h b/llvm/include/llvm/Object/OffloadBinary.h
index c63ef4824b..ee01cabe14 100644
--- a/llvm/include/llvm/Object/OffloadBinary.h
+++ b/llvm/include/llvm/Object/OffloadBinary.h
@@ -17,12 +17,12 @@
#ifndef LLVM_OBJECT_OFFLOADBINARY_H
#define LLVM_OBJECT_OFFLOADBINARY_H
-#include "llvm/Support/Compression.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Compression.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include <memory>
@@ -218,24 +218,30 @@ private:
int64_t NumberOfEntries;
public:
-
struct BundleEntry {
uint64_t Offset = 0u;
uint64_t Size = 0u;
uint64_t IDLength = 0u;
StringRef ID;
- BundleEntry(uint64_t O, uint64_t S, uint64_t I, StringRef T )
- : Offset(O), Size(S), IDLength(I), ID(T) {}
- void dump(raw_ostream &OS) { OS << "Offset = " << Offset << ", Size = " << Size << ", ID Length = " << IDLength << ", ID = " << ID;}
- void dumpURI(raw_ostream &OS, StringRef filePath) {OS << ID.data() << "\tfile:\/\/" << filePath << "#offset=" << Offset << "&size=" << Size <<"\n";}
+ BundleEntry(uint64_t O, uint64_t S, uint64_t I, StringRef T)
+ : Offset(O), Size(S), IDLength(I), ID(T) {}
+ void dump(raw_ostream &OS) {
+ OS << "Offset = " << Offset << ", Size = " << Size
+ << ", ID Length = " << IDLength << ", ID = " << ID;
+ }
+ void dumpURI(raw_ostream &OS, StringRef filePath) {
+ OS << ID.data() << "\tfile:\/\/" << filePath << "#offset=" << Offset
+ << "&size=" << Size << "\n";
+ }
};
uint64_t getSize() const { return Size; }
- StringRef getFileName() const {return FileName;}
- int64_t getNumEntries() const { return NumberOfEntries;}
+ StringRef getFileName() const { return FileName; }
+ int64_t getNumEntries() const { return NumberOfEntries; }
std::unique_ptr<SmallVector<BundleEntry>> Entries;
- static Expected<std::unique_ptr<OffloadFatBinBundle>> create(MemoryBufferRef, uint64_t SectionOffset, StringRef fileName);
+ static Expected<std::unique_ptr<OffloadFatBinBundle>>
+ create(MemoryBufferRef, uint64_t SectionOffset, StringRef fileName);
Error extractBundle(const ObjectFile &Source);
Error ReadEntries(StringRef Section, uint64_t SectionOffset);
@@ -250,19 +256,18 @@ public:
void PrintEntriesAsURI() {
SmallVectorImpl<BundleEntry>::iterator it = Entries->begin();
for (int64_t I = 0; I < NumberOfEntries; I++) {
- it->dumpURI(outs(),FileName);
+ it->dumpURI(outs(), FileName);
++it;
}
}
- OffloadFatBinBundle(MemoryBufferRef Source, StringRef file) :
- FileName(file) {
+ OffloadFatBinBundle(MemoryBufferRef Source, StringRef file) : FileName(file) {
NumberOfEntries = 0;
Entries = std::make_unique<SmallVector<BundleEntry>>();
}
};
-enum uri_type_t {FILE_URI, MEMORY_URI};
+enum uri_type_t { FILE_URI, MEMORY_URI };
struct OffloadBundleURI {
int64_t Offset = 0;
@@ -273,39 +278,43 @@ struct OffloadBundleURI {
// Constructors
// TODO: add a Copy ctor ?
- OffloadBundleURI(StringRef file, int64_t off, int64_t size) : Offset(off), Size(size), ProcessID(0), FileName(file), URIType(FILE_URI) {}
+ OffloadBundleURI(StringRef file, int64_t off, int64_t size)
+ : Offset(off), Size(size), ProcessID(0), FileName(file),
+ URIType(FILE_URI) {}
OffloadBundleURI(StringRef str, uri_type_t type) {
URIType = type;
switch (URIType) {
- case FILE_URI:
- parseFileName(str);
- break;
- case MEMORY_URI:
- parseMemoryURI(str);
- break;
- default:
- report_fatal_error("Unrecognized URI type.");
- }
+ case FILE_URI:
+ parseFileName(str);
+ break;
+ case MEMORY_URI:
+ parseMemoryURI(str);
+ break;
+ default:
+ report_fatal_error("Unrecognized URI type.");
+ }
}
void parseFileName(StringRef str) {
- ProcessID=0;
- URIType=FILE_URI;
+ ProcessID = 0;
+ URIType = FILE_URI;
if (str.consume_front("file://")) {
- StringRef FilePathname = str.take_until([](char c) {return (c == '#') || (c == '?');});
+ StringRef FilePathname =
+ str.take_until([](char c) { return (c == '#') || (c == '?'); });
FileName = FilePathname;
str = str.drop_front(FilePathname.size());
if (str.consume_front("#offset=")) {
- StringRef OffsetStr = str.take_until([](char c) { return c == '&';});
- OffsetStr.getAsInteger(10,Offset);
+ StringRef OffsetStr = str.take_until([](char c) { return c == '&'; });
+ OffsetStr.getAsInteger(10, Offset);
str = str.drop_front(OffsetStr.size());
if (str.consume_front("&size=")) {
- Size; str.getAsInteger(10,Size);
+ Size;
+ str.getAsInteger(10, Size);
} else
- report_fatal_error("Reading 'size' in URI.");
+ report_fatal_error("Reading 'size' in URI.");
} else
report_fatal_error("Reading 'offset' in URI.");
} else
@@ -316,7 +325,7 @@ struct OffloadBundleURI {
// TODO: add parseMemoryURI type
}
- StringRef getFileName() const { return FileName;}
+ StringRef getFileName() const { return FileName; }
};
/// Extracts embedded device offloading code from a memory \p Buffer to a list
@@ -324,9 +333,11 @@ struct OffloadBundleURI {
Error extractOffloadBinaries(MemoryBufferRef Buffer,
SmallVectorImpl<OffloadFile> &Binaries);
-Error extractFatBinaryFromObject(const ObjectFile &Obj, SmallVectorImpl<OffloadFatBinBundle> &Bundles);
+Error extractFatBinaryFromObject(const ObjectFile &Obj,
+ SmallVectorImpl<OffloadFatBinBundle> &Bundles);
-Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size, StringRef OutputFileName);
+Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size,
+ StringRef OutputFileName);
Error extractURI(StringRef URIstr);
diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp
index 75085b96a9..5b30fcbdc1 100644
--- a/llvm/lib/Object/OffloadBinary.cpp
+++ b/llvm/lib/Object/OffloadBinary.cpp
@@ -9,6 +9,7 @@
#include "llvm/Object/OffloadBinary.h"
#include "llvm/ADT/StringSwitch.h"
+#include "llvm/BinaryFormat/COFF.h"
#include "llvm/BinaryFormat/Magic.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Module.h"
@@ -17,17 +18,16 @@
#include "llvm/Object/Archive.h"
#include "llvm/Object/ArchiveWriter.h"
#include "llvm/Object/Binary.h"
-#include "llvm/BinaryFormat/COFF.h"
#include "llvm/Object/COFF.h"
#include "llvm/Object/ELFObjectFile.h"
#include "llvm/Object/Error.h"
#include "llvm/Object/IRObjectFile.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Alignment.h"
+#include "llvm/Support/BinaryStreamReader.h"
#include "llvm/Support/FileOutputBuffer.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/Timer.h"
-#include "llvm/Support/BinaryStreamReader.h"
using namespace llvm;
using namespace llvm::object;
@@ -106,8 +106,11 @@ Error extractFromObject(const ObjectFile &Obj,
return Error::success();
}
-// Extract an Offload bundle (usually a Clang Offload Bundle) from a fat_bin section
-Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset, StringRef fileName, SmallVectorImpl<OffloadFatBinBundle> &Bundles ) {
+// Extract an Offload bundle (usually a Clang Offload Bundle) from a fat_bin
+// section
+Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset,
+ StringRef fileName,
+ SmallVectorImpl<OffloadFatBinBundle> &Bundles) {
uint64_t Offset = 0;
int64_t nextbundleStart = 0;
@@ -119,18 +122,21 @@ Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset, Str
MemoryBuffer::getMemBuffer(Contents.getBuffer().drop_front(Offset), "",
/*RequiresNullTerminator*/ false);
- // Create the FatBinBindle object. This will also create the Bundle Entry list info.
- auto FatBundleOrErr = OffloadFatBinBundle::create(*Buffer, SectionOffset + Offset, fileName);
+ // Create the FatBinBindle object. This will also create the Bundle Entry
+ // list info.
+ auto FatBundleOrErr =
+ OffloadFatBinBundle::create(*Buffer, SectionOffset + Offset, fileName);
if (!FatBundleOrErr)
return FatBundleOrErr.takeError();
OffloadFatBinBundle &Bundle = **FatBundleOrErr;
// add current Bundle to list.
- Bundles.emplace_back(std::move(**FatBundleOrErr));
+ Bundles.emplace_back(std::move(**FatBundleOrErr));
// find the next bundle by searching for the magic string
StringRef str = Buffer->getBuffer();
- nextbundleStart = (int64_t)str.find(StringRef("__CLANG_OFFLOAD_BUNDLE__"), 24);
+ nextbundleStart =
+ (int64_t)str.find(StringRef("__CLANG_OFFLOAD_BUNDLE__"), 24);
if (nextbundleStart >= 0)
Offset += nextbundleStart;
@@ -213,61 +219,65 @@ Error extractFromArchive(const Archive &Library,
} // namespace
-
-Error OffloadFatBinBundle::ReadEntries(StringRef Buffer, uint64_t SectionOffset) {
+Error OffloadFatBinBundle::ReadEntries(StringRef Buffer,
+ uint64_t SectionOffset) {
uint64_t BundleNumber = 0;
uint64_t NumOfEntries = 0;
// get Reader
- BinaryStreamReader Reader( Buffer, llvm::endianness::little);
+ BinaryStreamReader Reader(Buffer, llvm::endianness::little);
// Read the Magic String first.
StringRef Magic;
if (auto EC = Reader.readFixedString(Magic, 24)) {
- return errorCodeToError(object_error::parse_failed);
- }
+ return errorCodeToError(object_error::parse_failed);
+ }
- // read the number of Code Objects (Entries) in the current Bundle.
+ // read the number of Code Objects (Entries) in the current Bundle.
if (auto EC = Reader.readInteger(NumOfEntries)) {
- printf("OffloadFatBinBundle::ReadEntries .... failed to read number of Entries\n");
+ printf("OffloadFatBinBundle::ReadEntries .... failed to read number of "
+ "Entries\n");
return errorCodeToError(object_error::parse_failed);
}
NumberOfEntries = NumOfEntries;
- // For each Bundle Entry (code object)
- for (uint64_t I = 0; I < NumOfEntries; I++) {
- uint64_t EntrySize;
- uint64_t EntryOffset;
- uint64_t EntryIDSize;
- StringRef EntryID;
- uint64_t absOffset;
+ // For each Bundle Entry (code object)
+ for (uint64_t I = 0; I < NumOfEntries; I++) {
+ uint64_t EntrySize;
+ uint64_t EntryOffset;
+ uint64_t EntryIDSize;
+ StringRef EntryID;
+ uint64_t absOffset;
- if (auto EC = Reader.readInteger(EntryOffset)) {
- return errorCodeToError(object_error::parse_failed);
- }
+ if (auto EC = Reader.readInteger(EntryOffset)) {
+ return errorCodeToError(object_error::parse_failed);
+ }
- if (auto EC = Reader.readInteger(EntrySize)) {
- return errorCodeToError(object_error::parse_failed);
- }
+ if (auto EC = Reader.readInteger(EntrySize)) {
+ return errorCodeToError(object_error::parse_failed);
+ }
- if (auto EC = Reader.readInteger(EntryIDSize)) {
- return errorCodeToError(object_error::parse_failed);
- }
+ if (auto EC = Reader.readInteger(EntryIDSize)) {
+ return errorCodeToError(object_error::parse_failed);
+ }
- if (auto EC = Reader.readFixedString(EntryID, EntryIDSize)) {
- return errorCodeToError(object_error::parse_failed);
- }
+ if (auto EC = Reader.readFixedString(EntryID, EntryIDSize)) {
+ return errorCodeToError(object_error::parse_failed);
+ }
- // create a Bundle Entry object:
- auto entry = new OffloadFatBinBundle::BundleEntry(EntryOffset+SectionOffset, EntrySize, EntryIDSize, EntryID);
+ // create a Bundle Entry object:
+ auto entry = new OffloadFatBinBundle::BundleEntry(
+ EntryOffset + SectionOffset, EntrySize, EntryIDSize, EntryID);
- Entries->push_back(*entry);
- } // end of for loop
+ Entries->push_back(*entry);
+ } // end of for loop
- return Error::success();
+ return Error::success();
}
-Expected<std::unique_ptr<OffloadFatBinBundle>> OffloadFatBinBundle::create(MemoryBufferRef Buf, uint64_t SectionOffset, StringRef fileName) {
+Expected<std::unique_ptr<OffloadFatBinBundle>>
+OffloadFatBinBundle::create(MemoryBufferRef Buf, uint64_t SectionOffset,
+ StringRef fileName) {
if (Buf.getBufferSize() < 24)
return errorCodeToError(object_error::parse_failed);
@@ -275,7 +285,7 @@ Expected<std::unique_ptr<OffloadFatBinBundle>> OffloadFatBinBundle::create(Memor
if (identify_magic(Buf.getBuffer()) != file_magic::offload_bundle)
return errorCodeToError(object_error::parse_failed);
- OffloadFatBinBundle* TheBundle = new OffloadFatBinBundle(Buf, fileName);
+ OffloadFatBinBundle *TheBundle = new OffloadFatBinBundle(Buf, fileName);
// Read the Bundle Entries
Error Err = TheBundle->ReadEntries(Buf.getBuffer(), SectionOffset);
@@ -287,14 +297,18 @@ Expected<std::unique_ptr<OffloadFatBinBundle>> OffloadFatBinBundle::create(Memor
Error OffloadFatBinBundle::extractBundle(const ObjectFile &Source) {
// This will extract all entries in the Bundle
- SmallVectorImpl<OffloadFatBinBundle::BundleEntry>::iterator it = Entries->begin();
+ SmallVectorImpl<OffloadFatBinBundle::BundleEntry>::iterator it =
+ Entries->begin();
for (int64_t I = 0; I < getNumEntries(); I++) {
- if (it->Size >0) {
- // create output file name. Which should be <fileName>-offset<Offset>-size<Size>.co"
- std::string str = getFileName().str() + "-offset" + itostr(it->Offset) + "-size" + itostr(it->Size) + ".co";
- if (Error Err = object::extractCodeObject(Source, it->Offset, it->Size, StringRef(str)))
- return Err;
+ if (it->Size > 0) {
+ // create output file name. Which should be
+ // <fileName>-offset<Offset>-size<Size>.co"
+ std::string str = getFileName().str() + "-offset" + itostr(it->Offset) +
+ "-size" + itostr(it->Size) + ".co";
+ if (Error Err = object::extractCodeObject(Source, it->Offset, it->Size,
+ StringRef(str)))
+ return Err;
}
++it;
}
@@ -431,7 +445,8 @@ Error object::extractOffloadBinaries(MemoryBufferRef Buffer,
}
}
-Error object::extractFatBinaryFromObject(const ObjectFile &Obj, SmallVectorImpl<OffloadFatBinBundle> &Bundles) {
+Error object::extractFatBinaryFromObject(
+ const ObjectFile &Obj, SmallVectorImpl<OffloadFatBinBundle> &Bundles) {
assert((Obj.isELF() || Obj.isCOFF()) && "Invalid file type");
// iterate through Sections until we find an offload_bundle section.
@@ -442,36 +457,43 @@ Error object::extractFatBinaryFromObject(const ObjectFile &Obj, SmallVectorImpl<
// If it does not start with the reserved suffix, just skip this section.
if ((llvm::identify_magic(*Buffer) == llvm::file_magic::offload_bundle) ||
- (llvm::identify_magic(*Buffer) == llvm::file_magic::offload_bundle_compressed)){
+ (llvm::identify_magic(*Buffer) ==
+ llvm::file_magic::offload_bundle_compressed)) {
uint64_t SectionOffset = 0;
if (Obj.isELF()) {
- SectionOffset = ELFSectionRef(Sec).getOffset();
+ SectionOffset = ELFSectionRef(Sec).getOffset();
} else if (Obj.isCOFF()) {
- if (const COFFObjectFile *COFFObj = dyn_cast<COFFObjectFile>(&Obj)) {
- const coff_section *CoffSection = COFFObj->getCOFFSection(Sec);
- fprintf(stderr,"DAVE: COFF viritual address =0x%llX\n", CoffSection->VirtualAddress);// COFFObj->getCOFFSection(Sec)->VirtualAddress);
- }
+ if (const COFFObjectFile *COFFObj = dyn_cast<COFFObjectFile>(&Obj)) {
+ const coff_section *CoffSection = COFFObj->getCOFFSection(Sec);
+ fprintf(
+ stderr, "DAVE: COFF viritual address =0x%llX\n",
+ CoffSection
+ ->VirtualAddress); // COFFObj->getCOFFSection(Sec)->VirtualAddress);
+ }
}
MemoryBufferRef Contents(*Buffer, Obj.getFileName());
- if (llvm::identify_magic(*Buffer) == llvm::file_magic::offload_bundle_compressed){
+ if (llvm::identify_magic(*Buffer) ==
+ llvm::file_magic::offload_bundle_compressed) {
// Decompress the input if necessary.
Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr =
CompressedOffloadBundle::decompress(Contents, false);
if (!DecompressedBufferOrErr)
- return createStringError(
- inconvertibleErrorCode(),
- "Failed to decompress input: " +
- llvm::toString(DecompressedBufferOrErr.takeError()));
-
- MemoryBuffer &DecompressedInput = **DecompressedBufferOrErr;
- if (Error Err = extractOffloadBundle(DecompressedInput, SectionOffset, Obj.getFileName() ,Bundles))
- return Err;
+ return createStringError(
+ inconvertibleErrorCode(),
+ "Failed to decompress input: " +
+ llvm::toString(DecompressedBufferOrErr.takeError()));
+
+ MemoryBuffer &DecompressedInput = **DecompressedBufferOrErr;
+ if (Error Err = extractOffloadBundle(DecompressedInput, SectionOffset,
+ Obj.getFileName(), Bundles))
+ return Err;
} else {
- if (Error Err = extractOffloadBundle(Contents, SectionOffset, Obj.getFileName() ,Bundles))
+ if (Error Err = extractOffloadBundle(Contents, SectionOffset,
+ Obj.getFileName(), Bundles))
return Err;
}
}
@@ -479,9 +501,10 @@ Error object::extractFatBinaryFromObject(const ObjectFile &Obj, SmallVectorImpl<
return Error::success();
}
-Error object::extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size, StringRef OutputFileName) {
+Error object::extractCodeObject(const ObjectFile &Source, int64_t Offset,
+ int64_t Size, StringRef OutputFileName) {
Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
- FileOutputBuffer::create(OutputFileName, Size);
+ FileOutputBuffer::create(OutputFileName, Size);
if (!BufferOrErr)
return BufferOrErr.takeError();
@@ -491,7 +514,9 @@ Error object::extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_
return Err;
std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
- std::copy(InputBuffOrErr->getBufferStart() + Offset, InputBuffOrErr->getBufferStart() + Offset + Size, Buf->getBufferStart());
+ std::copy(InputBuffOrErr->getBufferStart() + Offset,
+ InputBuffOrErr->getBufferStart() + Offset + Size,
+ Buf->getBufferStart());
if (Error E = Buf->commit())
return E;
@@ -502,18 +527,21 @@ Error object::extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_
// into file <SourceFile>-offset<Offset>-size<Size>.co
Error object::extractURI(StringRef URIstr) {
// create a URI object
- object::OffloadBundleURI* uri = new object::OffloadBundleURI(URIstr, FILE_URI);
+ object::OffloadBundleURI *uri =
+ new object::OffloadBundleURI(URIstr, FILE_URI);
std::string OutputFile = uri->FileName.str();
- OutputFile +="-offset" + itostr(uri->Offset) + "-size" + itostr(uri->Size) + ".co";
+ OutputFile +=
+ "-offset" + itostr(uri->Offset) + "-size" + itostr(uri->Size) + ".co";
// Create an ObjectFile object from uri.file_uri
auto ObjOrErr = ObjectFile::createObjectFile(uri->FileName);
- if(!ObjOrErr)
+ if (!ObjOrErr)
return ObjOrErr.takeError();
auto Obj = ObjOrErr->getBinary();
- if (Error Err = object::extractCodeObject(*Obj, uri->Offset, uri->Size, OutputFile))
+ if (Error Err =
+ object::extractCodeObject(*Obj, uri->Offset, uri->Size, OutputFile))
return Err;
return Error::success();
@@ -603,7 +631,6 @@ bool object::areTargetsCompatible(const OffloadFile::TargetID &LHS,
return true;
}
-
// Utility function to format numbers with commas
static std::string formatWithCommas(unsigned long long Value) {
std::string Num = std::to_string(Value);
``````````
</details>
https://github.com/llvm/llvm-project/pull/114833
More information about the llvm-commits
mailing list