[lld] [llvm] [DTLTO] Add DTLTO-specific LTO input handling time-trace scopes (PR #175799)
Ben Dunbobbin via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 16 08:26:52 PST 2026
https://github.com/bd1976bris updated https://github.com/llvm/llvm-project/pull/175799
>From 68ec8a978737aa8a760d8a6cde3e1c2399b1ec7c Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Tue, 13 Jan 2026 11:04:35 +0000
Subject: [PATCH 1/6] [DTLTO] Add DTLTO-specific LTO input handling time-trace
scopes
Add time-trace scopes to the DTLTO-specific input handling code to
improve observability and debugging.
These scopes are tested via LLD, as the primary purpose of this code
is to support member files of non-thin archives as DTLTO inputs.
`llvm-lto2` does not currently support archives. Adding archive
support to `llvm-lto2` solely for testing these scopes does not appear
to be worthwhile.
As part of this change I have moved when the DTLTO temporary input file
deletion occurs. It now happens after LTO is complete rather than when
the LTO object is destroyed, although I have left the clean-up code in
the destructor in case an error causes LTO to terminate prematurely.
At the point of LTO destruction the time traces have already been finalised
so no more trace data is processed. It's important to record time trace
data for the deletion of these files as this is an area where there have
been performance issues in the past and where we anticipate making performance
improvements in the future if the data supports it.
SIE internal tracker: TOOLCHAIN-21021
---
lld/test/ELF/dtlto/timetrace.test | 65 +++++++++++++++++++++++++++++++
llvm/include/llvm/DTLTO/DTLTO.h | 3 ++
llvm/include/llvm/LTO/LTO.h | 2 +
llvm/lib/DTLTO/DTLTO.cpp | 7 ++++
llvm/lib/LTO/LTO.cpp | 2 +
5 files changed, 79 insertions(+)
create mode 100644 lld/test/ELF/dtlto/timetrace.test
diff --git a/lld/test/ELF/dtlto/timetrace.test b/lld/test/ELF/dtlto/timetrace.test
new file mode 100644
index 0000000000000..e197b6f3b598b
--- /dev/null
+++ b/lld/test/ELF/dtlto/timetrace.test
@@ -0,0 +1,65 @@
+REQUIRES: x86
+
+## Test that DTLTO-specific LTO input file handling time-trace output is
+## produced as expected.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+
+## Create archives.
+RUN: llvm-ar rcs t1.a t1.bc
+RUN: llvm-ar rcsT t2.thin.a t2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Link and generate a time-trace.
+## Note: mock.py doesn't compile; it copies the specified object files to the
+## outputs in job order.
+RUN: ld.lld --whole-archive t1.a t2.thin.a -o my.elf \
+RUN: --thinlto-distributor=%python \
+RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN: --thinlto-distributor-arg=t1.o --thinlto-distributor-arg=t2.o \
+RUN: --time-trace-granularity=0 --time-trace=%t.json
+RUN: %python filter_order_and_pprint.py %t.json | FileCheck %s
+
+## Check that DTLTO add input file events are recorded.
+CHECK: "name": "Add input for DTLTO"
+CHECK: "name": "Add input for DTLTO"
+CHECK: "name": "Remove temporary inputs for DTLTO"
+CHECK: "name": "Save input archive member for DTLTO"
+CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[a-fA-F0-9]+]].o"
+CHECK: "name": "Total Add input for DTLTO"
+CHECK-SAME: "count": 2,
+CHECK: "name": "Total Remove temporary inputs for DTLTO"
+CHECK-SAME: "count": 1,
+CHECK: "name": "Total Save input archive member for DTLTO"
+CHECK-SAME: "count": 1,
+
+#--- t1.ll
+target triple = "x86_64-unknown-linux-gnu"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @t1() {
+ ret void
+}
+
+#--- filter_order_and_pprint.py
+import json, sys
+
+data = json.load(open(sys.argv[1], "r", encoding="utf-8"))
+
+# Get DTLTO events.
+events = [e for e in data["traceEvents"] if "DTLTO" in e["name"]]
+events.sort(key=lambda e: (e["name"], str(e.get("args", {}).get("detail", ""))))
+
+# Print an event per line. Ensure 'name' is the first key.
+for ev in events:
+ name = ev.pop("name")
+ print(json.dumps({"name": name, **ev}))
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index f9fad743e721c..f24276ece120d 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -54,6 +54,9 @@ class DTLTO : public LTO {
// Entry point for DTLTO archives support.
LLVM_ABI virtual llvm::Error handleArchiveInputs() override;
+
+ // Remove the temporary DTLTO input files.
+ LLVM_ABI virtual void cleanup() override;
};
} // namespace lto
} // namespace llvm
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 819be1909ec12..c10a906a46166 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -622,6 +622,8 @@ class LTO {
}
virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); }
+
+ virtual void cleanup() {}
};
/// The resolution for a symbol. The linker must provide a SymbolResolution for
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index 7ba4bfd80b6ab..ac3ebe84cd39c 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -25,6 +25,7 @@
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
+#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
#include <iostream>
@@ -119,6 +120,7 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) {
// Removes any temporary regular archive member files that were created during
// processing.
void lto::DTLTO::removeTempFiles() {
+ TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
for (auto &Input : InputFiles) {
if (Input->isMemberOfArchive())
sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
@@ -133,6 +135,7 @@ void lto::DTLTO::removeTempFiles() {
// 4. Updates the bitcode module's identifier.
Expected<std::shared_ptr<lto::InputFile>>
lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
+ TimeTraceScope TimeScope("Add input for DTLTO");
// Add the input file to the LTO object.
InputFiles.emplace_back(InputPtr.release());
@@ -180,6 +183,7 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) {
StringRef ModuleId = Input->getName();
if (Input->isMemberOfArchive()) {
+ TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId);
MemoryBufferRef MemoryBufferRef = Input->getFileBuffer();
if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId))
return EC;
@@ -210,3 +214,6 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
return EC;
return Error::success();
}
+
+// Cleanup temporary files after LTO is complete.
+void lto::DTLTO::cleanup() { removeTempFiles(); }
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index ff6762ebb59be..606d92c772a86 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1272,6 +1272,8 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
if (StatsFile)
PrintStatisticsJSON(StatsFile->os());
+ cleanup();
+
return Result;
}
>From 657c85b4926d0d62f0d404e5fc7dd82e4bb7e4e5 Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Fri, 16 Jan 2026 10:12:23 +0000
Subject: [PATCH 2/6] Addressed review comments
---
lld/test/ELF/dtlto/timetrace.test | 2 +-
llvm/include/llvm/DTLTO/DTLTO.h | 3 +++
llvm/lib/DTLTO/DTLTO.cpp | 3 +++
3 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/lld/test/ELF/dtlto/timetrace.test b/lld/test/ELF/dtlto/timetrace.test
index e197b6f3b598b..639ad36f8019f 100644
--- a/lld/test/ELF/dtlto/timetrace.test
+++ b/lld/test/ELF/dtlto/timetrace.test
@@ -34,7 +34,7 @@ CHECK: "name": "Add input for DTLTO"
CHECK: "name": "Add input for DTLTO"
CHECK: "name": "Remove temporary inputs for DTLTO"
CHECK: "name": "Save input archive member for DTLTO"
-CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[a-fA-F0-9]+]].o"
+CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[A-F0-9]+]].o"
CHECK: "name": "Total Add input for DTLTO"
CHECK-SAME: "count": 2,
CHECK: "name": "Total Remove temporary inputs for DTLTO"
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index f24276ece120d..cdae411fb29d7 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -29,6 +29,9 @@ class DTLTO : public LTO {
// Removes temporary files.
LLVM_ABI void removeTempFiles();
+ // Internal bookeeping.
+ bool RemoveTempFiles = true;
+
// Determines if a file at the given path is a thin archive file.
Expected<bool> isThinArchive(const StringRef ArchivePath);
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index ac3ebe84cd39c..27a247b318d19 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -120,6 +120,9 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) {
// Removes any temporary regular archive member files that were created during
// processing.
void lto::DTLTO::removeTempFiles() {
+ if (!RemoveTempFiles)
+ return;
+
TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
for (auto &Input : InputFiles) {
if (Input->isMemberOfArchive())
>From d8c3759153c33b50c1baf275ddde44223889a3db Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Fri, 16 Jan 2026 10:27:14 +0000
Subject: [PATCH 3/6] Use a scoped exit to call cleanup
---
llvm/include/llvm/DTLTO/DTLTO.h | 5 +----
llvm/lib/DTLTO/DTLTO.cpp | 3 ---
llvm/lib/LTO/LTO.cpp | 5 +++--
3 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index cdae411fb29d7..38c54e5d181ef 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -19,7 +19,7 @@ class DTLTO : public LTO {
public:
// Inherit contructors from LTO base class.
using LTO::LTO;
- ~DTLTO() { removeTempFiles(); }
+ ~DTLTO() {}
private:
// Bump allocator for a purpose of saving updated module IDs.
@@ -29,9 +29,6 @@ class DTLTO : public LTO {
// Removes temporary files.
LLVM_ABI void removeTempFiles();
- // Internal bookeeping.
- bool RemoveTempFiles = true;
-
// Determines if a file at the given path is a thin archive file.
Expected<bool> isThinArchive(const StringRef ArchivePath);
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index 27a247b318d19..ac3ebe84cd39c 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -120,9 +120,6 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) {
// Removes any temporary regular archive member files that were created during
// processing.
void lto::DTLTO::removeTempFiles() {
- if (!RemoveTempFiles)
- return;
-
TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
for (auto &Input : InputFiles) {
if (Input->isMemberOfArchive())
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 606d92c772a86..c5c6645e10a9d 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1215,6 +1215,9 @@ Error LTO::checkPartiallySplit() {
}
Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
+
+ llvm::scope_exit CleanUp([this]() { cleanup(); });
+
if (Error EC = handleArchiveInputs())
return EC;
@@ -1272,8 +1275,6 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
if (StatsFile)
PrintStatisticsJSON(StatsFile->os());
- cleanup();
-
return Result;
}
>From 09cda99280b2d9b424796c13f2d99bf033cf1ec2 Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Fri, 16 Jan 2026 10:36:30 +0000
Subject: [PATCH 4/6] Improve comment
---
llvm/lib/DTLTO/DTLTO.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index ac3ebe84cd39c..fb40d52b78553 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -215,5 +215,5 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
return Error::success();
}
-// Cleanup temporary files after LTO is complete.
+// Cleanup after LTO is complete.
void lto::DTLTO::cleanup() { removeTempFiles(); }
>From ff09db9af95736f2dbf3617e97cd14a18c159864 Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Fri, 16 Jan 2026 10:44:16 +0000
Subject: [PATCH 5/6] Call the base class cleanup function from the derived
one. There are cleaner ways of doing this but I don't think they are
justified in this case.
---
llvm/lib/DTLTO/DTLTO.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index fb40d52b78553..fbe0a6e75acec 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -216,4 +216,7 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
}
// Cleanup after LTO is complete.
-void lto::DTLTO::cleanup() { removeTempFiles(); }
+void lto::DTLTO::cleanup() {
+ removeTempFiles();
+ LTO::cleanup();
+}
>From 3a1f26fb26af94b5db6594a3f938b6e55084cf59 Mon Sep 17 00:00:00 2001
From: Ben <ben.dunbobbin at sony.com>
Date: Fri, 16 Jan 2026 16:15:34 +0000
Subject: [PATCH 6/6] Further improvements.
Inlined and removed the now unused DTLTO::removeTempFiles().
Made handleArchiveInputs() and cleanup() protected members.
Moved DTLTO::addInput() declaration next to the other public
functions and gave it a more pithy comment (the definition has
a comment with all the details).
---
llvm/include/llvm/DTLTO/DTLTO.h | 32 +++++++++++++++-----------------
llvm/include/llvm/LTO/LTO.h | 11 +++++++----
llvm/lib/DTLTO/DTLTO.cpp | 22 +++++++++-------------
3 files changed, 31 insertions(+), 34 deletions(-)
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index 38c54e5d181ef..2d47067deda67 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -16,18 +16,27 @@ namespace llvm {
namespace lto {
class DTLTO : public LTO {
+ using Base = LTO;
+
public:
- // Inherit contructors from LTO base class.
- using LTO::LTO;
- ~DTLTO() {}
+ // Inherit constructors.
+ using Base::Base;
+ ~DTLTO() override = default;
+
+ // Add an input file and prepare it for distribution.
+ LLVM_ABI Expected<std::shared_ptr<InputFile>>
+ addInput(std::unique_ptr<InputFile> InputPtr) override;
+
+protected:
+ LLVM_ABI llvm::Error handleArchiveInputs() override;
+
+ LLVM_ABI void cleanup() override;
private:
// Bump allocator for a purpose of saving updated module IDs.
BumpPtrAllocator PtrAlloc;
StringSaver Saver{PtrAlloc};
- // Removes temporary files.
- LLVM_ABI void removeTempFiles();
// Determines if a file at the given path is a thin archive file.
Expected<bool> isThinArchive(const StringRef ArchivePath);
@@ -45,19 +54,8 @@ class DTLTO : public LTO {
// A cache to avoid repeatedly reading the same archive file.
StringMap<bool> ArchiveFiles;
-public:
- // Adds the input file to the LTO object's list of input files.
- // For archive members, generates a new module ID which is a path to a real
- // file on a filesystem.
- LLVM_ABI virtual Expected<std::shared_ptr<lto::InputFile>>
- addInput(std::unique_ptr<lto::InputFile> InputPtr) override;
-
- // Entry point for DTLTO archives support.
- LLVM_ABI virtual llvm::Error handleArchiveInputs() override;
-
- // Remove the temporary DTLTO input files.
- LLVM_ABI virtual void cleanup() override;
};
+
} // namespace lto
} // namespace llvm
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index c10a906a46166..81a82e5fcde0a 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -443,6 +443,13 @@ class LTO {
LLVM_ABI static SmallVector<const char *>
getRuntimeLibcallSymbols(const Triple &TT);
+protected:
+ // Finalise inputs before run().
+ virtual Error handleArchiveInputs() { return Error::success(); }
+
+ // Clean up after run().
+ virtual void cleanup() {}
+
private:
Config Conf;
@@ -620,10 +627,6 @@ class LTO {
addInput(std::unique_ptr<lto::InputFile> InputPtr) {
return std::shared_ptr<lto::InputFile>(InputPtr.release());
}
-
- virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); }
-
- virtual void cleanup() {}
};
/// The resolution for a symbol. The linker must provide a SymbolResolution for
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index fbe0a6e75acec..6425c0ef47c9c 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -117,16 +117,6 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) {
return IsThin;
}
-// Removes any temporary regular archive member files that were created during
-// processing.
-void lto::DTLTO::removeTempFiles() {
- TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
- for (auto &Input : InputFiles) {
- if (Input->isMemberOfArchive())
- sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
- }
-}
-
// This function performs the following tasks:
// 1. Adds the input file to the LTO object's list of input files.
// 2. For thin archive members, generates a new module ID which is a path to a
@@ -215,8 +205,14 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
return Error::success();
}
-// Cleanup after LTO is complete.
+// Remove temporary archive member files created to enable distribution.
void lto::DTLTO::cleanup() {
- removeTempFiles();
- LTO::cleanup();
+ {
+ TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
+ for (auto &Input : InputFiles)
+ if (Input->isMemberOfArchive())
+ sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
+ }
+ Base::cleanup();
}
+
More information about the llvm-commits
mailing list