[clang] e12905b - [OpenMP] Add basic support for properly handling static libraries
Joseph Huber via cfe-commits
cfe-commits at lists.llvm.org
Fri May 6 08:21:10 PDT 2022
Author: Joseph Huber
Date: 2022-05-06T11:20:58-04:00
New Revision: e12905b4d5f95952600515901b82da0a8eefcd73
URL: https://github.com/llvm/llvm-project/commit/e12905b4d5f95952600515901b82da0a8eefcd73
DIFF: https://github.com/llvm/llvm-project/commit/e12905b4d5f95952600515901b82da0a8eefcd73.diff
LOG: [OpenMP] Add basic support for properly handling static libraries
Currently we handle static libraries like any other object in the
linker wrapper. However, this does not preserve the sematnics that
dictate static libraries should be lazily loaded as the symbols are
needed. This allows us to ignore linking in architectures that are not
used by the main application being compiled. This patch adds the basic
support for detecting if a file came from a static library, and only
including it in the link job if it's used by other object files.
This patch only adds the basic support, to be more correct we should
check the symbols and only inclue the library if the link job contains
symbols that are needed. Ideally we could just put this on the linker
itself, but nvlink doesn't seem to support `.a` files.
Reviewed By: jdoerfert
Differential Revision: https://reviews.llvm.org/D125092
Added:
Modified:
clang/test/Driver/linker-wrapper.c
clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
Removed:
################################################################################
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index a180a792284eb..342822ff46637 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -48,3 +48,15 @@
// RUN: /usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA_OMP_LINK
// CUDA_OMP_LINK: nvlink{{.*}}-m64 -o {{.*}}.out -arch sm_70 {{.*}}.o {{.*}}.o
+
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t-lib.o \
+// RUN: -fembed-offload-object=%S/Inputs/dummy-elf.o,openmp,nvptx64-nvida-cuda,sm_70 \
+// RUN: -fembed-offload-object=%S/Inputs/dummy-elf.o,openmp,nvptx64-nvida-cuda,sm_52
+// RUN: llvm-ar rcs %t.a %t-lib.o
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t-obj.o \
+// RUN: -fembed-offload-object=%S/Inputs/dummy-elf.o,openmp,nvptx64-nvida-cuda,sm_70
+// RUN: clang-linker-wrapper --host-triple x86_64-unknown-linux-gnu --dry-run -linker-path \
+// RUN: /usr/bin/ld -- %t.a %t-obj.o -o a.out 2>&1 | FileCheck %s --check-prefix=STATIC-LIBRARY
+
+// STATIC-LIBRARY: nvlink{{.*}} -arch sm_70
+// STATIC-LIBRARY-NOT: nvlink{{.*}} -arch sm_50
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index d9a7fecdde24f..b9555df3d16f6 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -167,13 +167,15 @@ static constexpr unsigned FatbinaryOffset = 0x50;
/// Information for a device offloading file extracted from the host.
struct DeviceFile {
DeviceFile(StringRef Kind, StringRef TheTriple, StringRef Arch,
- StringRef Filename)
- : Kind(Kind), TheTriple(TheTriple), Arch(Arch), Filename(Filename) {}
+ StringRef Filename, bool IsLibrary = false)
+ : Kind(Kind), TheTriple(TheTriple), Arch(Arch), Filename(Filename),
+ IsLibrary(IsLibrary) {}
std::string Kind;
std::string TheTriple;
std::string Arch;
std::string Filename;
+ bool IsLibrary;
};
namespace llvm {
@@ -208,7 +210,8 @@ namespace {
Expected<Optional<std::string>>
extractFromBuffer(std::unique_ptr<MemoryBuffer> Buffer,
- SmallVectorImpl<DeviceFile> &DeviceFiles);
+ SmallVectorImpl<DeviceFile> &DeviceFiles,
+ bool IsLibrary = false);
void printCommands(ArrayRef<StringRef> CmdArgs) {
if (CmdArgs.empty())
@@ -324,7 +327,8 @@ void removeFromCompilerUsed(Module &M, GlobalValue &Value) {
/// buffer \p Contents. The buffer is expected to contain a valid offloading
/// binary format.
Error extractOffloadFiles(StringRef Contents, StringRef Prefix,
- SmallVectorImpl<DeviceFile> &DeviceFiles) {
+ SmallVectorImpl<DeviceFile> &DeviceFiles,
+ bool IsLibrary = false) {
uint64_t Offset = 0;
// There could be multiple offloading binaries stored at this section.
while (Offset < Contents.size()) {
@@ -361,7 +365,7 @@ Error extractOffloadFiles(StringRef Contents, StringRef Prefix,
return E;
DeviceFiles.emplace_back(Kind, Binary.getTriple(), Binary.getArch(),
- TempFile);
+ TempFile, IsLibrary);
Offset += Binary.getSize();
}
@@ -371,7 +375,8 @@ Error extractOffloadFiles(StringRef Contents, StringRef Prefix,
Expected<Optional<std::string>>
extractFromBinary(const ObjectFile &Obj,
- SmallVectorImpl<DeviceFile> &DeviceFiles) {
+ SmallVectorImpl<DeviceFile> &DeviceFiles,
+ bool IsLibrary = false) {
StringRef Extension = sys::path::extension(Obj.getFileName()).drop_front();
StringRef Prefix = sys::path::stem(Obj.getFileName());
SmallVector<StringRef, 4> ToBeStripped;
@@ -386,7 +391,8 @@ extractFromBinary(const ObjectFile &Obj,
if (!Contents)
return Contents.takeError();
- if (Error Err = extractOffloadFiles(*Contents, Prefix, DeviceFiles))
+ if (Error Err =
+ extractOffloadFiles(*Contents, Prefix, DeviceFiles, IsLibrary))
return std::move(Err);
ToBeStripped.push_back(*Name);
@@ -447,7 +453,8 @@ extractFromBinary(const ObjectFile &Obj,
Expected<Optional<std::string>>
extractFromBitcode(std::unique_ptr<MemoryBuffer> Buffer,
- SmallVectorImpl<DeviceFile> &DeviceFiles) {
+ SmallVectorImpl<DeviceFile> &DeviceFiles,
+ bool IsLibrary = false) {
LLVMContext Context;
SMDiagnostic Err;
std::unique_ptr<Module> M = getLazyIRModule(std::move(Buffer), Err, Context);
@@ -473,7 +480,8 @@ extractFromBitcode(std::unique_ptr<MemoryBuffer> Buffer,
StringRef Contents = CDS->getAsString();
- if (Error Err = extractOffloadFiles(Contents, Prefix, DeviceFiles))
+ if (Error Err =
+ extractOffloadFiles(Contents, Prefix, DeviceFiles, IsLibrary))
return std::move(Err);
ToBeDeleted.push_back(&GV);
@@ -521,7 +529,8 @@ extractFromArchive(const Archive &Library,
std::unique_ptr<MemoryBuffer> ChildBuffer =
MemoryBuffer::getMemBuffer(*ChildBufferRefOrErr, false);
- auto FileOrErr = extractFromBuffer(std::move(ChildBuffer), DeviceFiles);
+ auto FileOrErr = extractFromBuffer(std::move(ChildBuffer), DeviceFiles,
+ /*IsLibrary*/ true);
if (!FileOrErr)
return FileOrErr.takeError();
@@ -573,11 +582,11 @@ extractFromArchive(const Archive &Library,
/// device code stripped from the buffer will be returned.
Expected<Optional<std::string>>
extractFromBuffer(std::unique_ptr<MemoryBuffer> Buffer,
- SmallVectorImpl<DeviceFile> &DeviceFiles) {
+ SmallVectorImpl<DeviceFile> &DeviceFiles, bool IsLibrary) {
file_magic Type = identify_magic(Buffer->getBuffer());
switch (Type) {
case file_magic::bitcode:
- return extractFromBitcode(std::move(Buffer), DeviceFiles);
+ return extractFromBitcode(std::move(Buffer), DeviceFiles, IsLibrary);
case file_magic::elf_relocatable:
case file_magic::macho_object:
case file_magic::coff_object: {
@@ -585,7 +594,7 @@ extractFromBuffer(std::unique_ptr<MemoryBuffer> Buffer,
ObjectFile::createObjectFile(*Buffer, Type);
if (!ObjFile)
return ObjFile.takeError();
- return extractFromBinary(*ObjFile->get(), DeviceFiles);
+ return extractFromBinary(*ObjFile->get(), DeviceFiles, IsLibrary);
}
case file_magic::archive: {
Expected<std::unique_ptr<llvm::object::Archive>> LibFile =
@@ -1127,8 +1136,22 @@ Error linkDeviceFiles(ArrayRef<DeviceFile> DeviceFiles,
SmallVectorImpl<std::string> &LinkedImages) {
// Get the list of inputs for a specific device.
DenseMap<DeviceFile, SmallVector<std::string, 4>> LinkerInputMap;
- for (auto &File : DeviceFiles)
- LinkerInputMap[File].push_back(File.Filename);
+ SmallVector<DeviceFile, 4> LibraryFiles;
+ for (auto &File : DeviceFiles) {
+ if (File.IsLibrary)
+ LibraryFiles.push_back(File);
+ else
+ LinkerInputMap[File].push_back(File.Filename);
+ }
+
+ // Static libraries are loaded lazily as-needed, only add them if other files
+ // are present.
+ // TODO: We need to check the symbols as well, static libraries are only
+ // loaded if they contain symbols that are currently undefined or common
+ // in the symbol table.
+ for (auto &File : LibraryFiles)
+ if (LinkerInputMap.count(File))
+ LinkerInputMap[File].push_back(File.Filename);
// Try to link each device toolchain.
for (auto &LinkerInput : LinkerInputMap) {
More information about the cfe-commits
mailing list