[lld] 14e3bec - Reland "[lld] Preliminary fat-lto-object support"
Paul Kirth via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 18 15:51:33 PDT 2023
Author: Paul Kirth
Date: 2023-08-18T22:51:25Z
New Revision: 14e3bec8fc3e1f10c3dc57277ae3dbf9a4087b1c
URL: https://github.com/llvm/llvm-project/commit/14e3bec8fc3e1f10c3dc57277ae3dbf9a4087b1c
DIFF: https://github.com/llvm/llvm-project/commit/14e3bec8fc3e1f10c3dc57277ae3dbf9a4087b1c.diff
LOG: Reland "[lld] Preliminary fat-lto-object support"
This patch adds support to lld for --fat-lto-objects. We add a new
--fat-lto-objects option to LLD, and slightly change how it chooses input
files in the driver when the option is set.
Fat LTO objects contain both LTO compatible IR, as well as generated object
code. This allows users to defer the choice of whether to use LTO or not to
link-time. This is a feature available in GCC for some time, and makes the
existing -ffat-lto-objects option functional in the same way as GCC's.
If the --fat-lto-objects option is passed to LLD and the input files are fat
object files, then the linker will chose the LTO compatible bitcode sections
embedded within the fat object and link them together using LTO. Otherwise,
standard object file linking is done using the assembly section in the object
files.
The previous version of this patch had a missing `REQUIRES: x86` line in
`fatlto.invalid.s`. Additionally, it was reported that this patch caused
a test failure in `export-dynamic-symbols.s`, however,
29112a994694baee070a2021e00f772f1913d214 disabled the
`export-dynamic-symbols.s` test on Windows due to a quotation difference
between platforms, unrelated to this patch.
Original RFC: https://discourse.llvm.org/t/rfc-ffat-lto-objects-support/63977
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D146778
Added:
lld/test/ELF/fatlto/fatlto.invalid.s
lld/test/ELF/fatlto/fatlto.test
Modified:
lld/ELF/Config.h
lld/ELF/Driver.cpp
lld/ELF/Options.td
lld/docs/ReleaseNotes.rst
lld/docs/ld.lld.1
Removed:
################################################################################
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index bbf2d201564581..6395befad53b87 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -125,7 +125,8 @@ class LinkerDriver {
void inferMachineType();
void link(llvm::opt::InputArgList &args);
template <class ELFT> void compileBitcodeFiles(bool skipLinkedOutput);
-
+ bool tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
+ uint64_t offsetInArchive, bool lazy);
// True if we are in --whole-archive and --no-whole-archive.
bool inWholeArchive = false;
@@ -205,6 +206,7 @@ struct Config {
callGraphProfile;
bool cmseImplib = false;
bool allowMultipleDefinition;
+ bool fatLTOObjects;
bool androidPackDynRelocs = false;
bool armHasBlx = false;
bool armHasMovtMovw = false;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 1759598b6d9705..ce35f993971cd5 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -52,6 +52,7 @@
#include "llvm/Config/llvm-config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
+#include "llvm/Object/IRObjectFile.h"
#include "llvm/Remarks/HotnessThresholdParser.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compression.h"
@@ -239,6 +240,19 @@ static bool isBitcode(MemoryBufferRef mb) {
return identify_magic(mb.getBuffer()) == llvm::file_magic::bitcode;
}
+bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
+ uint64_t offsetInArchive, bool lazy) {
+ if (!config->fatLTOObjects)
+ return false;
+ Expected<MemoryBufferRef> fatLTOData =
+ IRObjectFile::findBitcodeInMemBuffer(mb);
+ if (errorToBool(fatLTOData.takeError()))
+ return false;
+ files.push_back(
+ make<BitcodeFile>(*fatLTOData, archiveName, offsetInArchive, lazy));
+ return true;
+}
+
// Opens a file and create a file object. Path has to be resolved already.
void LinkerDriver::addFile(StringRef path, bool withLOption) {
using namespace sys::fs;
@@ -263,7 +277,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
if (isBitcode(p.first))
files.push_back(make<BitcodeFile>(p.first, path, p.second, false));
- else
+ else if (!tryAddFatLTOFile(p.first, path, p.second, false))
files.push_back(createObjFile(p.first, path));
}
return;
@@ -287,9 +301,10 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
InputFile::isInGroup = true;
for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
auto magic = identify_magic(p.first.getBuffer());
- if (magic == file_magic::elf_relocatable)
- files.push_back(createObjFile(p.first, path, true));
- else if (magic == file_magic::bitcode)
+ if (magic == file_magic::elf_relocatable) {
+ if (!tryAddFatLTOFile(p.first, path, p.second, true))
+ files.push_back(createObjFile(p.first, path, true));
+ } else if (magic == file_magic::bitcode)
files.push_back(make<BitcodeFile>(p.first, path, p.second, true));
else
warn(path + ": archive member '" + p.first.getBufferIdentifier() +
@@ -321,7 +336,8 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
files.push_back(make<BitcodeFile>(mbref, "", 0, inLib));
break;
case file_magic::elf_relocatable:
- files.push_back(createObjFile(mbref, "", inLib));
+ if (!tryAddFatLTOFile(mbref, "", 0, inLib))
+ files.push_back(createObjFile(mbref, "", inLib));
break;
default:
error(path + ": unknown file type");
@@ -1134,6 +1150,8 @@ static void readConfigs(opt::InputArgList &args) {
args.hasFlag(OPT_android_memtag_heap, OPT_no_android_memtag_heap, false);
config->androidMemtagStack = args.hasFlag(OPT_android_memtag_stack,
OPT_no_android_memtag_stack, false);
+ config->fatLTOObjects =
+ args.hasFlag(OPT_fat_lto_objects, OPT_no_fat_lto_objects, false);
config->androidMemtagMode = getMemtagMode(args);
config->auxiliaryList = args::getStrings(args, OPT_auxiliary);
config->armBe8 = args.hasArg(OPT_be8);
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 0d5c6c3d80a106..e91e08989853c7 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -647,6 +647,10 @@ def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
+defm fat_lto_objects: BB<"fat-lto-objects",
+ "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
+ "Ignore the .llvm.lto section in relocatable object files (default).">;
+
def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
def: F<"plugin-opt=debug-pass-manager">,
Alias<lto_debug_pass_manager>, HelpText<"Alias for --lto-debug-pass-manager">;
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 6f60efd87c975b..1590879416853e 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -26,6 +26,11 @@ Non-comprehensive list of changes in this release
ELF Improvements
----------------
+* ``--fat-lto-objects`` option is added to support LLVM FatLTO.
+ Without ``--fat-lto-objects``, LLD will link LLVM FatLTO objects using the
+ relocatable object file. (`D146778 <https://reviews.llvm.org/D146778>`_)
+
+
Breaking changes
----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 0a5e4293dedaf5..97ba66abd3eba8 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -621,6 +621,10 @@ Number of threads.
(default) means all of concurrent threads supported.
.Cm 1
disables multi-threading.
+.It Fl -fat-lto-objects
+Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.
+.It Fl -no-fat-lto-objects
+Ignore the .llvm.lto section in relocatable object files (default).
.It Fl -time-trace
Record time trace.
.It Fl -time-trace-file Ns = Ns Ar file
diff --git a/lld/test/ELF/fatlto/fatlto.invalid.s b/lld/test/ELF/fatlto/fatlto.invalid.s
new file mode 100644
index 00000000000000..a712b1dbb58b48
--- /dev/null
+++ b/lld/test/ELF/fatlto/fatlto.invalid.s
@@ -0,0 +1,10 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
+# RUN: not ld.lld %t -o /dev/null --fat-lto-objects 2>&1 | FileCheck %s
+
+# CHECK: error:{{.*}} Invalid bitcode signature
+
+.section .llvm.lto,"e", at progbits
+.Lllvm.embedded.object:
+ .asciz "BC\300\3365\000"
+ .size .Lllvm.embedded.object, 12
diff --git a/lld/test/ELF/fatlto/fatlto.test b/lld/test/ELF/fatlto/fatlto.test
new file mode 100644
index 00000000000000..edf4ef2da2b888
--- /dev/null
+++ b/lld/test/ELF/fatlto/fatlto.test
@@ -0,0 +1,99 @@
+;; Basic FatLTO tests.
+; REQUIRES: x86
+
+; RUN: rm -rf %t && split-file %s %t
+
+;; Ensure that input files contain .llvm.lto section.
+; RUN: llc %t/a-LTO.ll --filetype=obj -o %t/a-fatLTO.o
+; RUN: opt < %t/a-LTO.ll --module-summary -o %t/a-fatLTO.bc
+; RUN: llvm-objcopy --add-section=.llvm.lto=%t/a-fatLTO.bc --set-section-flags=.llvm.lto=exclude %t/a-fatLTO.o
+
+; RUN: llc %t/main-LTO.ll --filetype=obj -o %t/main-fatLTO.o
+; RUN: opt < %t/main-LTO.ll --module-summary -o %t/main-fatLTO.bc
+; RUN: llvm-objcopy --add-section=.llvm.lto=%t/main-fatLTO.bc --set-section-flags=.llvm.lto=exclude %t/main-fatLTO.o
+
+;; Final executable should not have .llvm.lto section no matter what the target is.
+; RUN: ld.lld -o %t/foo-fatLTO %t/a-fatLTO.o %t/main-fatLTO.o --fat-lto-objects
+; RUN: llvm-readobj -S %t/foo-fatLTO | FileCheck --check-prefix=CHECK-LTO-TARGET %s
+
+;; Check that fat objects work w/ --start-lib.
+; RUN: ld.lld -o %t/foo-fatLTO.start_lib --start-lib %t/a-fatLTO.o %t/main-fatLTO.o --fat-lto-objects
+; RUN: llvm-readobj -S %t/foo-fatLTO.start_lib | FileCheck --check-prefix=CHECK-LTO-TARGET %s
+
+;; Check if .llvm.lto section gets aggregated in LTO target.
+; CHECK-LTO-TARGET-NOT: Name: .llvm.lto
+
+;; Final executable should not have .llvm.lto section no matter what the target is.
+; RUN: ld.lld -o %t/foo-fatNoLTO %t/a-fatLTO.o %/t/main-fatLTO.o
+; RUN: llvm-readobj -S %t/foo-fatNoLTO | FileCheck --check-prefix=CHECK-NON-LTO-TARGET %s
+
+;; Check if .llvm.lto section gets aggregated in non-LTO target.
+; CHECK-NON-LTO-TARGET-NOT: Name: .llvm.lto
+
+;; Check if the LTO target executable produced from FatLTO object file is
+;; identical to the one produced from LTO modules.
+; RUN: opt < %t/a-LTO.ll --module-summary -o %t/a-LTO.bc
+; RUN: opt < %t/main-LTO.ll --module-summary -o %t/main-LTO.bc
+; RUN: ld.lld -o %t/foo-LTO %t/a-LTO.bc %t/main-LTO.bc
+; RUN: cmp %t/foo-fatLTO %t/foo-LTO
+
+;; Check if the no-LTO target executable produced from FatLTO object file is
+;; identical to the one produced from regular object files.
+; RUN: llc %t/a-LTO.ll --filetype=obj -o %t/a.o
+; RUN: llc %t/main-LTO.ll --filetype=obj -o %t/main.o
+; RUN: ld.lld -o %t/foo-noLTO %t/a.o %t/main.o
+; RUN: cmp %t/foo-fatNoLTO %t/foo-noLTO
+
+;; Check archive support.
+; RUN: llvm-ar rcs %t/a.a %t/a-fatLTO.o
+; RUN: ld.lld -o %t/foo-fatLTO.archive %t/a.a %t/main-LTO.bc --fat-lto-objects
+; RUN: cmp %t/foo-fatLTO.archive %t/foo-LTO
+
+;--- a-LTO.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @_start() #0 {
+entry:
+ ret i32 0
+}
+
+attributes #0 = { noinline nounwind uwtable }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 2}
+!5 = !{i32 1, !"ThinLTO", i32 0}
+!6 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+
+;--- main-LTO.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @main() #0 {
+entry:
+ %retval = alloca i32, align 4
+ store i32 0, ptr %retval, align 4
+ %call = call i32 (...) @_start()
+ ret i32 %call
+}
+
+declare i32 @_start(...)
+
+attributes #0 = { noinline nounwind uwtable }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{i32 7, !"frame-pointer", i32 2}
+!5 = !{i32 1, !"ThinLTO", i32 0}
+!6 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
More information about the llvm-commits
mailing list