[lld] 3a45b84 - [lld] Preliminary fat-lto-object support
Paul Kirth via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 19 16:07:52 PDT 2023
Author: Paul Kirth
Date: 2023-07-19T23:07:42Z
New Revision: 3a45b843dec1bca195884aa1c5bc56bd0e6755b4
URL: https://github.com/llvm/llvm-project/commit/3a45b843dec1bca195884aa1c5bc56bd0e6755b4
DIFF: https://github.com/llvm/llvm-project/commit/3a45b843dec1bca195884aa1c5bc56bd0e6755b4.diff
LOG: [lld] Preliminary fat-lto-object support
This patch adds support to lld for --fat-lto-objects. We add a new
--fat-lto-objects flag to LLD, and slightly change how it chooses input
files in the driver when the flag is set.
Fat LTO objects contain both LTO compatible IR, as well as generated object
code. This allows users to defer the choice of whether to use LTO or not to
link-time. This is a feature available in GCC for some time, and makes the
existing -ffat-lto-objects flag functional in the same way as GCC's.
If the --fat-lto-objects option is passed to LLD and the input files are fat
object files, then the linker will chose the LTO compatible bitcode sections
embedded within the fat object and link them together using LTO. Otherwise,
standard object file linking is done using the assembly section in the object
files.
Original RFC: https://discourse.llvm.org/t/rfc-ffat-lto-objects-support/63977
Depends on D146777
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D146778
Added:
lld/test/ELF/fatlto/fatlto.invalid.s
lld/test/ELF/fatlto/fatlto.test
Modified:
lld/ELF/Config.h
lld/ELF/Driver.cpp
lld/ELF/Options.td
lld/docs/ReleaseNotes.rst
lld/docs/ld.lld.1
Removed:
################################################################################
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index bbf2d201564581..6395befad53b87 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -125,7 +125,8 @@ class LinkerDriver {
void inferMachineType();
void link(llvm::opt::InputArgList &args);
template <class ELFT> void compileBitcodeFiles(bool skipLinkedOutput);
-
+ bool tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
+ uint64_t offsetInArchive, bool lazy);
// True if we are in --whole-archive and --no-whole-archive.
bool inWholeArchive = false;
@@ -205,6 +206,7 @@ struct Config {
callGraphProfile;
bool cmseImplib = false;
bool allowMultipleDefinition;
+ bool fatLTOObjects;
bool androidPackDynRelocs = false;
bool armHasBlx = false;
bool armHasMovtMovw = false;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 4e3ab2cc425a14..5fc6684780cf22 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -52,6 +52,7 @@
#include "llvm/Config/llvm-config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
+#include "llvm/Object/IRObjectFile.h"
#include "llvm/Remarks/HotnessThresholdParser.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compression.h"
@@ -237,6 +238,19 @@ static bool isBitcode(MemoryBufferRef mb) {
return identify_magic(mb.getBuffer()) == llvm::file_magic::bitcode;
}
+bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
+ uint64_t offsetInArchive, bool lazy) {
+ if (!config->fatLTOObjects)
+ return false;
+ Expected<MemoryBufferRef> fatLTOData =
+ IRObjectFile::findBitcodeInMemBuffer(mb);
+ if (errorToBool(fatLTOData.takeError()))
+ return false;
+ files.push_back(
+ make<BitcodeFile>(*fatLTOData, archiveName, offsetInArchive, lazy));
+ return true;
+}
+
// Opens a file and create a file object. Path has to be resolved already.
void LinkerDriver::addFile(StringRef path, bool withLOption) {
using namespace sys::fs;
@@ -261,7 +275,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
if (isBitcode(p.first))
files.push_back(make<BitcodeFile>(p.first, path, p.second, false));
- else
+ else if (!tryAddFatLTOFile(p.first, path, p.second, false))
files.push_back(createObjFile(p.first, path));
}
return;
@@ -285,9 +299,10 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
InputFile::isInGroup = true;
for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
auto magic = identify_magic(p.first.getBuffer());
- if (magic == file_magic::elf_relocatable)
- files.push_back(createObjFile(p.first, path, true));
- else if (magic == file_magic::bitcode)
+ if (magic == file_magic::elf_relocatable) {
+ if (!tryAddFatLTOFile(p.first, path, p.second, true))
+ files.push_back(createObjFile(p.first, path, true));
+ } else if (magic == file_magic::bitcode)
files.push_back(make<BitcodeFile>(p.first, path, p.second, true));
else
warn(path + ": archive member '" + p.first.getBufferIdentifier() +
@@ -319,7 +334,8 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
files.push_back(make<BitcodeFile>(mbref, "", 0, inLib));
break;
case file_magic::elf_relocatable:
- files.push_back(createObjFile(mbref, "", inLib));
+ if (!tryAddFatLTOFile(mbref, "", 0, inLib))
+ files.push_back(createObjFile(mbref, "", inLib));
break;
default:
error(path + ": unknown file type");
@@ -1138,6 +1154,8 @@ static void readConfigs(opt::InputArgList &args) {
args.hasFlag(OPT_android_memtag_heap, OPT_no_android_memtag_heap, false);
config->androidMemtagStack = args.hasFlag(OPT_android_memtag_stack,
OPT_no_android_memtag_stack, false);
+ config->fatLTOObjects =
+ args.hasFlag(OPT_fat_lto_objects, OPT_no_fat_lto_objects, false);
config->androidMemtagMode = getMemtagMode(args);
config->auxiliaryList = args::getStrings(args, OPT_auxiliary);
config->armBe8 = args.hasArg(OPT_be8);
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 0d5c6c3d80a106..e91e08989853c7 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -647,6 +647,10 @@ def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
+defm fat_lto_objects: BB<"fat-lto-objects",
+ "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
+ "Ignore the .llvm.lto section in relocatable object files (default).">;
+
def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
def: F<"plugin-opt=debug-pass-manager">,
Alias<lto_debug_pass_manager>, HelpText<"Alias for --lto-debug-pass-manager">;
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 9435f23a7af616..52b8c238981307 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -31,6 +31,10 @@ ELF Improvements
* ``PT_RISCV_ATTRIBUTES`` is added to include the SHT_RISCV_ATTRIBUTES section.
(`D152065 <https://reviews.llvm.org/D152065>`_)
+- ``--fat-lto-objects`` option is added to support LLVM FatLTO.
+ Without ``--fat-lto-objects``, LLD will link LLVM FatLTO objects using the
+ relocatable object file. (`D146778 <https://reviews.llvm.org/D146778>`_)
+
Breaking changes
----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 898ec30dfe6f39..41cc09d96a3fcb 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -621,6 +621,10 @@ Number of threads.
(default) means all of concurrent threads supported.
.Cm 1
disables multi-threading.
+.It Fl -fat-lto-objects
+Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.
+.It Fl -no-fat-lto-objects
+Ignore the .llvm.lto section in relocatable object files (default).
.It Fl -time-trace
Record time trace.
.It Fl -time-trace-file Ns = Ns Ar file
diff --git a/lld/test/ELF/fatlto/fatlto.invalid.s b/lld/test/ELF/fatlto/fatlto.invalid.s
new file mode 100644
index 00000000000000..cc7f98d3c9b3e4
--- /dev/null
+++ b/lld/test/ELF/fatlto/fatlto.invalid.s
@@ -0,0 +1,9 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
+# RUN: not ld.lld %t -o /dev/null --fat-lto-objects 2>&1 | FileCheck %s
+
+# CHECK: error:{{.*}} Invalid bitcode signature
+
+.section .llvm.lto,"e", at progbits
+.Lllvm.embedded.object:
+ .asciz "BC\300\3365\000"
+ .size .Lllvm.embedded.object, 12
diff --git a/lld/test/ELF/fatlto/fatlto.test b/lld/test/ELF/fatlto/fatlto.test
new file mode 100644
index 00000000000000..edf4ef2da2b888
--- /dev/null
+++ b/lld/test/ELF/fatlto/fatlto.test
@@ -0,0 +1,99 @@
+;; Basic FatLTO tests.
+; REQUIRES: x86
+
+; RUN: rm -rf %t && split-file %s %t
+
+;; Ensure that input files contain .llvm.lto section.
+; RUN: llc %t/a-LTO.ll --filetype=obj -o %t/a-fatLTO.o
+; RUN: opt < %t/a-LTO.ll --module-summary -o %t/a-fatLTO.bc
+; RUN: llvm-objcopy --add-section=.llvm.lto=%t/a-fatLTO.bc --set-section-flags=.llvm.lto=exclude %t/a-fatLTO.o
+
+; RUN: llc %t/main-LTO.ll --filetype=obj -o %t/main-fatLTO.o
+; RUN: opt < %t/main-LTO.ll --module-summary -o %t/main-fatLTO.bc
+; RUN: llvm-objcopy --add-section=.llvm.lto=%t/main-fatLTO.bc --set-section-flags=.llvm.lto=exclude %t/main-fatLTO.o
+
+;; Final executable should not have .llvm.lto section no matter what the target is.
+; RUN: ld.lld -o %t/foo-fatLTO %t/a-fatLTO.o %t/main-fatLTO.o --fat-lto-objects
+; RUN: llvm-readobj -S %t/foo-fatLTO | FileCheck --check-prefix=CHECK-LTO-TARGET %s
+
+;; Check that fat objects work w/ --start-lib.
+; RUN: ld.lld -o %t/foo-fatLTO.start_lib --start-lib %t/a-fatLTO.o %t/main-fatLTO.o --fat-lto-objects
+; RUN: llvm-readobj -S %t/foo-fatLTO.start_lib | FileCheck --check-prefix=CHECK-LTO-TARGET %s
+
+;; Check if .llvm.lto section gets aggregated in LTO target.
+; CHECK-LTO-TARGET-NOT: Name: .llvm.lto
+
+;; Final executable should not have .llvm.lto section no matter what the target is.
+; RUN: ld.lld -o %t/foo-fatNoLTO %t/a-fatLTO.o %/t/main-fatLTO.o
+; RUN: llvm-readobj -S %t/foo-fatNoLTO | FileCheck --check-prefix=CHECK-NON-LTO-TARGET %s
+
+;; Check if .llvm.lto section gets aggregated in non-LTO target.
+; CHECK-NON-LTO-TARGET-NOT: Name: .llvm.lto
+
+;; Check if the LTO target executable produced from FatLTO object file is
+;; identical to the one produced from LTO modules.
+; RUN: opt < %t/a-LTO.ll --module-summary -o %t/a-LTO.bc
+; RUN: opt < %t/main-LTO.ll --module-summary -o %t/main-LTO.bc
+; RUN: ld.lld -o %t/foo-LTO %t/a-LTO.bc %t/main-LTO.bc
+; RUN: cmp %t/foo-fatLTO %t/foo-LTO
+
+;; Check if the no-LTO target executable produced from FatLTO object file is
+;; identical to the one produced from regular object files.
+; RUN: llc %t/a-LTO.ll --filetype=obj -o %t/a.o
+; RUN: llc %t/main-LTO.ll --filetype=obj -o %t/main.o
+; RUN: ld.lld -o %t/foo-noLTO %t/a.o %t/main.o
+; RUN: cmp %t/foo-fatNoLTO %t/foo-noLTO
+
+;; Check archive support.
+; RUN: llvm-ar rcs %t/a.a %t/a-fatLTO.o
+; RUN: ld.lld -o %t/foo-fatLTO.archive %t/a.a %t/main-LTO.bc --fat-lto-objects
+; RUN: cmp %t/foo-fatLTO.archive %t/foo-LTO
+
+;--- a-LTO.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @_start() #0 {
+entry:
+ ret i32 0
+}
+
+attributes #0 = { noinline nounwind uwtable }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 2}
+!5 = !{i32 1, !"ThinLTO", i32 0}
+!6 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+
+;--- main-LTO.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @main() #0 {
+entry:
+ %retval = alloca i32, align 4
+ store i32 0, ptr %retval, align 4
+ %call = call i32 (...) @_start()
+ ret i32 %call
+}
+
+declare i32 @_start(...)
+
+attributes #0 = { noinline nounwind uwtable }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 2}
+!5 = !{i32 1, !"ThinLTO", i32 0}
+!6 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
More information about the llvm-commits
mailing list