[clang] 93d08ac - [clang-offload-wrapper] Add standard notes for ELF offload images
Vyacheslav Zakharin via cfe-commits
cfe-commits at lists.llvm.org
Mon Aug 16 13:20:19 PDT 2021
Author: Vyacheslav Zakharin
Date: 2021-08-16T13:09:01-07:00
New Revision: 93d08acaacec951dbb302f77eeae51974985b6b2
URL: https://github.com/llvm/llvm-project/commit/93d08acaacec951dbb302f77eeae51974985b6b2
DIFF: https://github.com/llvm/llvm-project/commit/93d08acaacec951dbb302f77eeae51974985b6b2.diff
LOG: [clang-offload-wrapper] Add standard notes for ELF offload images
The patch adds ELF notes into SHT_NOTE sections of ELF offload images
passed to clang-offload-wrapper.
The new notes use a null-terminated "LLVMOMPOFFLOAD" note name.
There are currently three types of notes:
VERSION: a string (not null-terminated) representing the ELF offload
image structure. The current version '1.0' does not put any restrictions
on the structure of the image. If we ever need to come up with a common
structure for ELF offload images (e.g. to be able to analyze the images
in libomptarget in some standard way), then we will introduce new versions.
PRODUCER: a vendor specific name of the producing toolchain.
Upstream LLVM uses "LLVM" (not null-terminated).
PRODUCER_VERSION: a vendor specific version of the producing toolchain.
Upstream LLVM uses LLVM_VERSION_STRING with optional <space> LLVM_REVISION.
All three notes are not mandatory currently.
Differential Revision: https://reviews.llvm.org/D99551
Added:
clang/test/Driver/Inputs/empty-elf-template.yaml
Modified:
clang/test/Driver/clang-offload-wrapper.c
clang/tools/clang-offload-wrapper/CMakeLists.txt
clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp
Removed:
################################################################################
diff --git a/clang/test/Driver/Inputs/empty-elf-template.yaml b/clang/test/Driver/Inputs/empty-elf-template.yaml
new file mode 100644
index 0000000000000..f77de07a430f6
--- /dev/null
+++ b/clang/test/Driver/Inputs/empty-elf-template.yaml
@@ -0,0 +1,5 @@
+--- !ELF
+FileHeader:
+ Class: ELFCLASS[[BITS]]
+ Data: ELFDATA2[[ENCODING]]
+ Type: ET_REL
diff --git a/clang/test/Driver/clang-offload-wrapper.c b/clang/test/Driver/clang-offload-wrapper.c
index 9a36559e34dd7..c671d88209744 100644
--- a/clang/test/Driver/clang-offload-wrapper.c
+++ b/clang/test/Driver/clang-offload-wrapper.c
@@ -19,9 +19,10 @@
//
// Check bitcode produced by the wrapper tool.
//
-// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.bc %t.tgt
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.bc %t.tgt 2>&1 | FileCheck %s --check-prefix ELF-WARNING
// RUN: llvm-dis %t.wrapper.bc -o - | FileCheck %s --check-prefix CHECK-IR
+// ELF-WARNING: is not an ELF image, so notes cannot be added to it.
// CHECK-IR: target triple = "x86_64-pc-linux-gnu"
// CHECK-IR-DAG: [[ENTTY:%.+]] = type { i8*, i8*, i{{32|64}}, i32, i32 }
@@ -53,3 +54,24 @@
// CHECK-IR: ret void
// CHECK-IR: declare void @__tgt_unregister_lib([[DESCTY]]*)
+
+// Check that clang-offload-wrapper adds LLVMOMPOFFLOAD notes
+// into the ELF offload images:
+// RUN: yaml2obj %S/Inputs/empty-elf-template.yaml -o %t.64le -DBITS=64 -DENCODING=LSB
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.elf64le.bc %t.64le
+// RUN: llvm-dis %t.wrapper.elf64le.bc -o - | FileCheck %s --check-prefix OMPNOTES
+// RUN: yaml2obj %S/Inputs/empty-elf-template.yaml -o %t.64be -DBITS=64 -DENCODING=MSB
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.elf64be.bc %t.64be
+// RUN: llvm-dis %t.wrapper.elf64be.bc -o - | FileCheck %s --check-prefix OMPNOTES
+// RUN: yaml2obj %S/Inputs/empty-elf-template.yaml -o %t.32le -DBITS=32 -DENCODING=LSB
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.elf32le.bc %t.32le
+// RUN: llvm-dis %t.wrapper.elf32le.bc -o - | FileCheck %s --check-prefix OMPNOTES
+// RUN: yaml2obj %S/Inputs/empty-elf-template.yaml -o %t.32be -DBITS=32 -DENCODING=MSB
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -o %t.wrapper.elf32be.bc %t.32be
+// RUN: llvm-dis %t.wrapper.elf32be.bc -o - | FileCheck %s --check-prefix OMPNOTES
+
+// There is no clean way for extracting the offload image
+// from the object file currently, so try to find
+// the inserted ELF notes in the device image variable's
+// initializer:
+// OMPNOTES: @{{.+}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"{{.*}}LLVMOMPOFFLOAD{{.*}}LLVMOMPOFFLOAD{{.*}}LLVMOMPOFFLOAD{{.*}}"
diff --git a/clang/tools/clang-offload-wrapper/CMakeLists.txt b/clang/tools/clang-offload-wrapper/CMakeLists.txt
index 8bcb46267a37c..144edf5ab60c0 100644
--- a/clang/tools/clang-offload-wrapper/CMakeLists.txt
+++ b/clang/tools/clang-offload-wrapper/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_LINK_COMPONENTS BitWriter Core Support TransformUtils)
+set(LLVM_LINK_COMPONENTS BitWriter Core Object Support TransformUtils)
add_clang_tool(clang-offload-wrapper
ClangOffloadWrapper.cpp
diff --git a/clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp b/clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp
index fff12716b6f0d..bbadd909089e3 100644
--- a/clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp
+++ b/clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp
@@ -17,27 +17,37 @@
#include "clang/Basic/Version.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/EndianStream.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/VCSRevision.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <cassert>
#include <cstdint>
+#define OPENMP_OFFLOAD_IMAGE_VERSION "1.0"
+
using namespace llvm;
+using namespace llvm::object;
static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
@@ -60,6 +70,12 @@ static cl::opt<std::string>
cl::desc("Target triple for the output module"),
cl::value_desc("triple"), cl::cat(ClangOffloadWrapperCategory));
+static cl::opt<bool> SaveTemps(
+ "save-temps",
+ cl::desc("Save temporary files that may be produced by the tool. "
+ "This option forces print-out of the temporary files' names."),
+ cl::Hidden);
+
namespace {
class BinaryWrapper {
@@ -70,6 +86,15 @@ class BinaryWrapper {
StructType *ImageTy = nullptr;
StructType *DescTy = nullptr;
+ std::string ToolName;
+ std::string ObjcopyPath;
+ // Temporary file names that may be created during adding notes
+ // to ELF offload images. Use -save-temps to keep them and also
+ // see their names. A temporary file's name includes the name
+ // of the original input ELF image, so you can easily match
+ // them, if you have multiple inputs.
+ std::vector<std::string> TempFiles;
+
private:
IntegerType *getSizeTTy() {
switch (M.getDataLayout().getPointerTypeSize(Type::getInt8PtrTy(C))) {
@@ -294,8 +319,61 @@ class BinaryWrapper {
}
public:
- BinaryWrapper(StringRef Target) : M("offload.wrapper.object", C) {
+ BinaryWrapper(StringRef Target, StringRef ToolName)
+ : M("offload.wrapper.object", C), ToolName(ToolName) {
M.setTargetTriple(Target);
+ // Look for llvm-objcopy in the same directory, from which
+ // clang-offload-wrapper is invoked. This helps OpenMP offload
+ // LIT tests.
+
+ // This just needs to be some symbol in the binary; C++ doesn't
+ // allow taking the address of ::main however.
+ void *P = (void *)(intptr_t)&Help;
+ std::string COWPath = sys::fs::getMainExecutable(ToolName.str().c_str(), P);
+ if (!COWPath.empty()) {
+ auto COWDir = sys::path::parent_path(COWPath);
+ ErrorOr<std::string> ObjcopyPathOrErr =
+ sys::findProgramByName("llvm-objcopy", {COWDir});
+ if (ObjcopyPathOrErr) {
+ ObjcopyPath = *ObjcopyPathOrErr;
+ return;
+ }
+
+ // Otherwise, look through PATH environment.
+ }
+
+ ErrorOr<std::string> ObjcopyPathOrErr =
+ sys::findProgramByName("llvm-objcopy");
+ if (!ObjcopyPathOrErr) {
+ WithColor::warning(errs(), ToolName)
+ << "cannot find llvm-objcopy[.exe] in PATH; ELF notes cannot be "
+ "added.\n";
+ return;
+ }
+
+ ObjcopyPath = *ObjcopyPathOrErr;
+ }
+
+ ~BinaryWrapper() {
+ if (TempFiles.empty())
+ return;
+
+ StringRef ToolNameRef(ToolName);
+ auto warningOS = [ToolNameRef]() -> raw_ostream & {
+ return WithColor::warning(errs(), ToolNameRef);
+ };
+
+ for (auto &F : TempFiles) {
+ if (SaveTemps) {
+ warningOS() << "keeping temporary file " << F << "\n";
+ continue;
+ }
+
+ auto EC = sys::fs::remove(F, false);
+ if (EC)
+ warningOS() << "cannot remove temporary file " << F << ": "
+ << EC.message().c_str() << "\n";
+ }
}
const Module &wrapBinaries(ArrayRef<ArrayRef<char>> Binaries) {
@@ -305,6 +383,205 @@ class BinaryWrapper {
createUnregisterFunction(Desc);
return M;
}
+
+ std::unique_ptr<MemoryBuffer> addELFNotes(std::unique_ptr<MemoryBuffer> Buf,
+ StringRef OriginalFileName) {
+ // Cannot add notes, if llvm-objcopy is not available.
+ //
+ // I did not find a clean way to add a new notes section into an existing
+ // ELF file. llvm-objcopy seems to recreate a new ELF from scratch,
+ // and we just try to use llvm-objcopy here.
+ if (ObjcopyPath.empty())
+ return Buf;
+
+ StringRef ToolNameRef(ToolName);
+
+ // Helpers to emit warnings.
+ auto warningOS = [ToolNameRef]() -> raw_ostream & {
+ return WithColor::warning(errs(), ToolNameRef);
+ };
+ auto handleErrorAsWarning = [&warningOS](Error E) {
+ logAllUnhandledErrors(std::move(E), warningOS());
+ };
+
+ Expected<std::unique_ptr<ObjectFile>> BinOrErr =
+ ObjectFile::createELFObjectFile(Buf->getMemBufferRef(),
+ /*InitContent=*/false);
+ if (Error E = BinOrErr.takeError()) {
+ consumeError(std::move(E));
+ // This warning is questionable, but let it be here,
+ // assuming that most OpenMP offload models use ELF offload images.
+ warningOS() << OriginalFileName
+ << " is not an ELF image, so notes cannot be added to it.\n";
+ return Buf;
+ }
+
+ // If we fail to add the note section, we just pass through the original
+ // ELF image for wrapping. At some point we should enforce the note section
+ // and start emitting errors vs warnings.
+ support::endianness Endianness;
+ if (isa<ELF64LEObjectFile>(BinOrErr->get()) ||
+ isa<ELF32LEObjectFile>(BinOrErr->get())) {
+ Endianness = support::little;
+ } else if (isa<ELF64BEObjectFile>(BinOrErr->get()) ||
+ isa<ELF32BEObjectFile>(BinOrErr->get())) {
+ Endianness = support::big;
+ } else {
+ warningOS() << OriginalFileName
+ << " is an ELF image of unrecognized format.\n";
+ return Buf;
+ }
+
+ // Create temporary file for the data of a new SHT_NOTE section.
+ // We fill it in with data and then pass to llvm-objcopy invocation
+ // for reading.
+ Twine NotesFileModel = OriginalFileName + Twine(".elfnotes.%%%%%%%.tmp");
+ Expected<sys::fs::TempFile> NotesTemp =
+ sys::fs::TempFile::create(NotesFileModel);
+ if (Error E = NotesTemp.takeError()) {
+ handleErrorAsWarning(createFileError(NotesFileModel, std::move(E)));
+ return Buf;
+ }
+ TempFiles.push_back(NotesTemp->TmpName);
+
+ // Create temporary file for the updated ELF image.
+ // This is an empty file that we pass to llvm-objcopy invocation
+ // for writing.
+ Twine ELFFileModel = OriginalFileName + Twine(".elfwithnotes.%%%%%%%.tmp");
+ Expected<sys::fs::TempFile> ELFTemp =
+ sys::fs::TempFile::create(ELFFileModel);
+ if (Error E = ELFTemp.takeError()) {
+ handleErrorAsWarning(createFileError(ELFFileModel, std::move(E)));
+ return Buf;
+ }
+ TempFiles.push_back(ELFTemp->TmpName);
+
+ // Keep the new ELF image file to reserve the name for the future
+ // llvm-objcopy invocation.
+ std::string ELFTmpFileName = ELFTemp->TmpName;
+ if (Error E = ELFTemp->keep(ELFTmpFileName)) {
+ handleErrorAsWarning(createFileError(ELFTmpFileName, std::move(E)));
+ return Buf;
+ }
+
+ // Write notes to the *elfnotes*.tmp file.
+ raw_fd_ostream NotesOS(NotesTemp->FD, false);
+
+ struct NoteTy {
+ // Note name is a null-terminated "LLVMOMPOFFLOAD".
+ std::string Name;
+ // Note type defined in llvm/include/llvm/BinaryFormat/ELF.h.
+ uint32_t Type = 0;
+ // Each note has type-specific associated data.
+ std::string Desc;
+
+ NoteTy(std::string &&Name, uint32_t Type, std::string &&Desc)
+ : Name(std::move(Name)), Type(Type), Desc(std::move(Desc)) {}
+ };
+
+ // So far we emit just three notes.
+ SmallVector<NoteTy, 3> Notes;
+ // Version of the offload image identifying the structure of the ELF image.
+ // Version 1.0 does not have any specific requirements.
+ // We may come up with some structure that has to be honored by all
+ // offload implementations in future (e.g. to let libomptarget
+ // get some information from the offload image).
+ Notes.emplace_back("LLVMOMPOFFLOAD", ELF::NT_LLVM_OPENMP_OFFLOAD_VERSION,
+ OPENMP_OFFLOAD_IMAGE_VERSION);
+ // This is a producer identification string. We are LLVM!
+ Notes.emplace_back("LLVMOMPOFFLOAD", ELF::NT_LLVM_OPENMP_OFFLOAD_PRODUCER,
+ "LLVM");
+ // This is a producer version. Use the same format that is used
+ // by clang to report the LLVM version.
+ Notes.emplace_back("LLVMOMPOFFLOAD",
+ ELF::NT_LLVM_OPENMP_OFFLOAD_PRODUCER_VERSION,
+ LLVM_VERSION_STRING
+#ifdef LLVM_REVISION
+ " " LLVM_REVISION
+#endif
+ );
+
+ // Return the amount of padding required for a blob of N bytes
+ // to be aligned to Alignment bytes.
+ auto getPadAmount = [](uint32_t N, uint32_t Alignment) -> uint32_t {
+ uint32_t Mod = (N % Alignment);
+ if (Mod == 0)
+ return 0;
+ return Alignment - Mod;
+ };
+ auto emitPadding = [&getPadAmount](raw_ostream &OS, uint32_t Size) {
+ for (uint32_t I = 0; I < getPadAmount(Size, 4); ++I)
+ OS << '\0';
+ };
+
+ // Put notes into the file.
+ for (auto &N : Notes) {
+ assert(!N.Name.empty() && "We should not create notes with empty names.");
+ // Name must be null-terminated.
+ if (N.Name.back() != '\0')
+ N.Name += '\0';
+ uint32_t NameSz = N.Name.size();
+ uint32_t DescSz = N.Desc.size();
+ // A note starts with three 4-byte values:
+ // NameSz
+ // DescSz
+ // Type
+ // These three fields are endian-sensitive.
+ support::endian::write<uint32_t>(NotesOS, NameSz, Endianness);
+ support::endian::write<uint32_t>(NotesOS, DescSz, Endianness);
+ support::endian::write<uint32_t>(NotesOS, N.Type, Endianness);
+ // Next, we have a null-terminated Name padded to a 4-byte boundary.
+ NotesOS << N.Name;
+ emitPadding(NotesOS, NameSz);
+ if (DescSz == 0)
+ continue;
+ // Finally, we have a descriptor, which is an arbitrary flow of bytes.
+ NotesOS << N.Desc;
+ emitPadding(NotesOS, DescSz);
+ }
+ NotesOS.flush();
+
+ // Keep the notes file.
+ std::string NotesTmpFileName = NotesTemp->TmpName;
+ if (Error E = NotesTemp->keep(NotesTmpFileName)) {
+ handleErrorAsWarning(createFileError(NotesTmpFileName, std::move(E)));
+ return Buf;
+ }
+
+ // Run llvm-objcopy like this:
+ // llvm-objcopy --add-section=.note.openmp=<notes-tmp-file-name> \
+ // <orig-file-name> <elf-tmp-file-name>
+ //
+ // This will add a SHT_NOTE section on top of the original ELF.
+ std::vector<StringRef> Args;
+ Args.push_back(ObjcopyPath);
+ std::string Option("--add-section=.note.openmp=" + NotesTmpFileName);
+ Args.push_back(Option);
+ Args.push_back(OriginalFileName);
+ Args.push_back(ELFTmpFileName);
+ bool ExecutionFailed = false;
+ std::string ErrMsg;
+ (void)sys::ExecuteAndWait(ObjcopyPath, Args,
+ /*Env=*/llvm::None, /*Redirects=*/{},
+ /*SecondsToWait=*/0,
+ /*MemoryLimit=*/0, &ErrMsg, &ExecutionFailed);
+
+ if (ExecutionFailed) {
+ warningOS() << ErrMsg << "\n";
+ return Buf;
+ }
+
+ // Substitute the original ELF with new one.
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+ MemoryBuffer::getFile(ELFTmpFileName);
+ if (!BufOrErr) {
+ handleErrorAsWarning(
+ createFileError(ELFTmpFileName, BufOrErr.getError()));
+ return Buf;
+ }
+
+ return std::move(*BufOrErr);
+ }
};
} // anonymous namespace
@@ -338,6 +615,8 @@ int main(int argc, const char **argv) {
return 1;
}
+ BinaryWrapper Wrapper(Target, argv[0]);
+
// Read device binaries.
SmallVector<std::unique_ptr<MemoryBuffer>, 4u> Buffers;
SmallVector<ArrayRef<char>, 4u> Images;
@@ -350,8 +629,13 @@ int main(int argc, const char **argv) {
reportError(createFileError(File, BufOrErr.getError()));
return 1;
}
+ std::unique_ptr<MemoryBuffer> Buffer(std::move(*BufOrErr));
+ if (File != "-") {
+ // Adding ELF notes for STDIN is not supported yet.
+ Buffer = Wrapper.addELFNotes(std::move(Buffer), File);
+ }
const std::unique_ptr<MemoryBuffer> &Buf =
- Buffers.emplace_back(std::move(*BufOrErr));
+ Buffers.emplace_back(std::move(Buffer));
Images.emplace_back(Buf->getBufferStart(), Buf->getBufferSize());
}
@@ -364,9 +648,9 @@ int main(int argc, const char **argv) {
}
// Create a wrapper for device binaries and write its bitcode to the file.
- WriteBitcodeToFile(BinaryWrapper(Target).wrapBinaries(
- makeArrayRef(Images.data(), Images.size())),
- Out.os());
+ WriteBitcodeToFile(
+ Wrapper.wrapBinaries(makeArrayRef(Images.data(), Images.size())),
+ Out.os());
if (Out.os().has_error()) {
reportError(createFileError(Output, Out.os().error()));
return 1;
More information about the cfe-commits
mailing list