[lld] [llvm] [DTLTO] support distributing bitcode from FatLTO objects (PR #176928)
Rose Hudson via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 20 06:21:58 PST 2026
https://github.com/rosefromthedead created https://github.com/llvm/llvm-project/pull/176928
We already have code to extract bitcode files from archives so they can be distributed. Extend this code to extract bitcode from FatLTO objects too, which otherwise cannot be used with DTLTO.
>From ced5e0ea431cce9f90d4a8ef29daaddcd49b5def Mon Sep 17 00:00:00 2001
From: Rose Hudson <rose.hudson at sony.com>
Date: Fri, 16 Jan 2026 14:14:22 +0000
Subject: [PATCH] [DTLTO] support distributing bitcode from FatLTO objects
We already have code to extract bitcode files from archives so they can
be distributed. Extend this code to extract bitcode from FatLTO objects
too, which otherwise cannot be used with DTLTO.
---
lld/ELF/Driver.cpp | 6 ++-
lld/test/ELF/dtlto/fat-lto-object.test | 61 ++++++++++++++++++++++++++
llvm/include/llvm/LTO/LTO.h | 16 ++++---
llvm/lib/DTLTO/DTLTO.cpp | 20 ++++-----
4 files changed, 86 insertions(+), 17 deletions(-)
create mode 100644 lld/test/ELF/dtlto/fat-lto-object.test
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 8647752be31fe..9944cf2e73700 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -236,8 +236,10 @@ bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
IRObjectFile::findBitcodeInMemBuffer(mb);
if (errorToBool(fatLTOData.takeError()))
return false;
- files.push_back(std::make_unique<BitcodeFile>(ctx, *fatLTOData, archiveName,
- offsetInArchive, lazy));
+ auto file = std::make_unique<BitcodeFile>(ctx, *fatLTOData, archiveName,
+ offsetInArchive, lazy);
+ file->obj->fatLTOObject(true);
+ files.push_back(std::move(file));
return true;
}
diff --git a/lld/test/ELF/dtlto/fat-lto-object.test b/lld/test/ELF/dtlto/fat-lto-object.test
new file mode 100644
index 0000000000000..dad46fc732435
--- /dev/null
+++ b/lld/test/ELF/dtlto/fat-lto-object.test
@@ -0,0 +1,61 @@
+REQUIRES: x86
+
+## Test that a DTLTO link can use FatLTO objects as inputs, even if they're in archives.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+RUN: sed 's/@t1/@t3/g' t1.ll > t3.ll
+
+## Taken from lld/test/ELF/fatlto/fatlto.test
+RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic
+RUN: opt < t1.ll --module-summary -o t1.bc
+RUN: llvm-objcopy --add-section=.llvm.lto=t1.bc --set-section-flags=.llvm.lto=exclude --set-section-type=.llvm.lto=0x6fff4c0c t1.o
+
+RUN: llc t2.ll --filetype=obj -o t2.o --relocation-model=pic
+RUN: opt < t2.ll --module-summary -o t2.bc
+RUN: llvm-objcopy --add-section=.llvm.lto=t2.bc --set-section-flags=.llvm.lto=exclude --set-section-type=.llvm.lto=0x6fff4c0c t2.o
+
+RUN: llc t3.ll --filetype=obj -o t3.o --relocation-model=pic
+RUN: opt < t3.ll --module-summary -o t3.bc
+RUN: llvm-objcopy --add-section=.llvm.lto=t3.bc --set-section-flags=.llvm.lto=exclude --set-section-type=.llvm.lto=0x6fff4c0c t3.o
+
+## Create thin and regular archives containing FatLTO objects to test that we
+## can combine these weird file types.
+RUN: llvm-ar rcsT t1.a t1.o
+RUN: llvm-ar rcs t2.a t2.o
+
+RUN: echo "t1.a t2.a t3.o \
+RUN: --thinlto-distributor=\"%python\" \
+RUN: --thinlto-distributor-arg=\"%llvm_src_root/utils/dtlto/validate.py\"" > args
+
+## Link thin archives using -u/--undefined.
+RUN: not ld.lld --fat-lto-objects @args -u t1 -u t2 2>&1
+
+## Link thin archives using --whole-archive.
+RUN: not ld.lld --fat-lto-objects --whole-archive @args 2>&1
+
+## Check the module IDs in the JSON jobs description.
+CHECK: "jobs": [
+CHECK: "inputs": [
+CHECK-NEXT: "{{([a-zA-Z]:)|/}}
+CHECK-SAME: t1.bc"
+
+CHECK: "inputs": [
+CHECK-NEXT: "{{([a-zA-Z]\:)|/}}
+CHECK-SAME: t2.bc"
+
+CHECK: "inputs": [
+CHECK-NEXT: "{{([a-zA-Z]:)|/}}
+CHECK-SAME: t3.bc"
+
+## Ensure backend compilation fails as expected (due to validate.py dummy behavior).
+CHECK: error: DTLTO backend compilation: cannot open native object file:
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+ ret void
+}
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index cba5cf7eb9e62..d51add9fa4230 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -131,7 +131,8 @@ class InputFile {
std::vector<std::pair<StringRef, Comdat::SelectionKind>> ComdatTable;
MemoryBufferRef MbRef;
- bool IsMemberOfArchive = false;
+ bool IsFatLTOObject = false;
+ bool IsEntireFile = true;
bool IsThinLTO = false;
StringRef ArchivePath;
StringRef MemberName;
@@ -198,10 +199,15 @@ class InputFile {
LLVM_ABI BitcodeModule &getPrimaryBitcodeModule();
// Returns the memory buffer reference for this input file.
MemoryBufferRef getFileBuffer() const { return MbRef; }
- // Returns true if this input file is a member of an archive.
- bool isMemberOfArchive() const { return IsMemberOfArchive; }
- // Mark this input file as a member of archive.
- void memberOfArchive(bool MA) { IsMemberOfArchive = MA; }
+ // Returns false if this input file is a member of an archive or of a FatLTO
+ // object.
+ bool isEntireFile() const { return IsEntireFile; }
+ // Mark this input file as being an entire file on disk.
+ void entireFile(bool EF) { IsEntireFile = EF; }
+ // Returns true if this bitcode came from a FatLTO object.
+ bool isFatLTOObject() const { return IsFatLTOObject; }
+ // Mark this bitcode as coming from a FatLTO object.
+ void fatLTOObject(bool FO) { IsFatLTOObject = FO; }
// Returns true if bitcode is ThinLTO.
bool isThinLTO() const { return IsThinLTO; }
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index b9a5cd3a062e2..bb99d3f3ca460 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -21,14 +21,12 @@
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
-#include <iostream>
#include <string>
using namespace llvm;
@@ -134,15 +132,17 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
StringRef ModuleId = Input->getName();
StringRef ArchivePath = Input->getArchivePath();
- // Only process archive members.
- if (ArchivePath.empty())
+ // Only process archive members and FatLTO objects.
+ if (ArchivePath.empty() && !Input->isFatLTOObject())
return Input;
SmallString<64> NewModuleId;
BitcodeModule &BM = Input->getPrimaryBitcodeModule();
- // Check if the archive is a thin archive.
- Expected<bool> IsThin = isThinArchive(ArchivePath);
+ // Check if the archive is a thin archive and the object is not fat.
+ // This means we have a file already that we can point to.
+ Expected<bool> IsThin =
+ Input->isFatLTOObject() ? false : isThinArchive(ArchivePath);
if (!IsThin)
return IsThin.takeError();
@@ -151,8 +151,8 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
NewModuleId =
computeThinArchiveMemberPath(ArchivePath, Input->getMemberName());
} else {
- // For regular archives, generate a unique name.
- Input->memberOfArchive(true);
+ // For regular archives and FatLTO objects, generate a unique name.
+ Input->entireFile(false);
// Create unique identifier using process ID and sequence number.
std::string PID = utohexstr(sys::Process::getProcessId());
@@ -172,7 +172,7 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
// previously terminated linker process and can be safely overwritten.
Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) {
StringRef ModuleId = Input->getName();
- if (Input->isMemberOfArchive()) {
+ if (!Input->isEntireFile()) {
TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId);
MemoryBufferRef MemoryBufferRef = Input->getFileBuffer();
if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId))
@@ -210,7 +210,7 @@ void lto::DTLTO::cleanup() {
{
TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
for (auto &Input : InputFiles)
- if (Input->isMemberOfArchive())
+ if (!Input->isEntireFile())
sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
}
Base::cleanup();
More information about the llvm-commits
mailing list