[lld] [llvm] [lld] Add caching for `--lto-partitions` (PR #212203)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 01:11:01 PDT 2026
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/212203
>From 8d991368b5a0313c717c44596e07006d0a056730 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Mon, 27 Jul 2026 10:57:49 +0200
Subject: [PATCH 1/2] [lld] Add caching for `--lto-partitions`
Add an opt-in cache for the (full) LTO partitions generated via `--lto-partitions=N` when `N > 1`.
This is primarily for AMDGPU but implemented in a target-agnostic way. The goal is to avoid recompiling entire very large modules (can be hundreds of megabytes of bitcode) if someone just changed a single function or two, which don't affect most partitions.
This uses the LTO Config hash + a hash of the bitcode module itself. This is a very conservative approach, we may be able to fine-tune it to improve cache hits if it turns out the hit rate is poor.
Solves LCOMPILER-59
---
lld/ELF/Config.h | 3 +
lld/ELF/Driver.cpp | 6 +
lld/ELF/LTO.cpp | 23 +++-
lld/ELF/Options.td | 5 +
.../ELF/lto/lto-partitions-cache-warnings.ll | 57 ++++++++
lld/test/ELF/lto/lto-partitions-cache.ll | 122 ++++++++++++++++++
...-warnings.ll => thinlto-cache-warnings.ll} | 0
.../ELF/lto/{cache.ll => thinlto-cache.ll} | 0
llvm/include/llvm/DTLTO/DTLTO.h | 11 +-
llvm/include/llvm/LTO/LTO.h | 25 +++-
llvm/include/llvm/LTO/LTOBackend.h | 1 +
.../llvm/LTO/legacy/LTOCodeGenerator.h | 83 ++++++++++++
llvm/lib/DTLTO/DTLTO.cpp | 3 +-
llvm/lib/LTO/LTO.cpp | 112 ++++++++++------
llvm/lib/LTO/LTOBackend.cpp | 68 +++++++++-
llvm/lib/LTO/LTOCodeGenerator.cpp | 30 ++++-
16 files changed, 487 insertions(+), 62 deletions(-)
create mode 100644 lld/test/ELF/lto/lto-partitions-cache-warnings.ll
create mode 100644 lld/test/ELF/lto/lto-partitions-cache.ll
rename lld/test/ELF/lto/{cache-warnings.ll => thinlto-cache-warnings.ll} (100%)
rename lld/test/ELF/lto/{cache.ll => thinlto-cache.ll} (100%)
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 54e0aa58591ad..3d853c425ee6f 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -489,6 +489,9 @@ struct Config {
int32_t splitStackAdjustSize;
SmallVector<uint8_t, 0> packageMetadata;
+ llvm::StringRef ltoPartitionsCacheDir;
+ llvm::CachePruningPolicy ltoPartitionsCachePolicy;
+
// The following config options do not directly correspond to any
// particular command line options.
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 0393410b35d4d..bcd7cf8da95a7 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1496,6 +1496,12 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
ErrAlways(ctx) << "invalid codegen optimization level for LTO: " << ltoCgo;
ctx.arg.ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
ctx.arg.ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
+ ctx.arg.ltoPartitionsCacheDir =
+ args.getLastArgValue(OPT_lto_partitions_cache_dir);
+ ctx.arg.ltoPartitionsCachePolicy =
+ CHECK(parseCachePruningPolicy(
+ args.getLastArgValue(OPT_lto_partitions_cache_pruning)),
+ "--lto-partitions-cache-pruning: invalid cache policy");
ctx.arg.ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
ctx.arg.ltoBBAddrMap =
args.hasFlag(OPT_lto_basic_block_address_map,
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index a11681133373d..cbe3c2552dca8 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -343,6 +343,17 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
cache = check(localCache("ThinLTO", "Thin", ctx.arg.thinLTOCacheDir,
createAddBufferFn(files, filenames)));
+ FileCache partitionsCache;
+ if (!ctx.arg.ltoPartitionsCacheDir.empty())
+ partitionsCache = check(localCache(
+ "LTOPartitions", "LTOPartition", ctx.arg.ltoPartitionsCacheDir,
+ [&](unsigned task, const Twine &moduleName,
+ std::unique_ptr<MemoryBuffer> mb) {
+ // TODO: Creates a copy for a bit, is that fine?
+ buf[task].second = mb->getBuffer();
+ buf[task].first = moduleName.str();
+ }));
+
if (!ctx.bitcodeFiles.empty())
checkError(ctx.e, ltoObj->run(
[&](size_t task, const Twine &moduleName) {
@@ -351,7 +362,7 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
std::make_unique<raw_svector_ostream>(
buf[task].second));
},
- cache));
+ cache, partitionsCache));
// Emit empty index files for non-indexed files but not in single-module mode.
if (ctx.arg.thinLTOModulesToCompile.empty()) {
@@ -382,6 +393,16 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
check(
pruneCache(ctx.arg.thinLTOCacheDir, ctx.arg.thinLTOCachePolicy, files));
+ if (!ctx.arg.ltoPartitionsCacheDir.empty()) {
+ std::vector<std::unique_ptr<MemoryBuffer>> ltoPartitionBuffers;
+ for (auto &e : buf) {
+ ltoPartitionBuffers.push_back(
+ MemoryBuffer::getMemBuffer(e.second, "", false));
+ }
+ check(pruneCache(ctx.arg.ltoPartitionsCacheDir,
+ ctx.arg.ltoPartitionsCachePolicy, ltoPartitionBuffers));
+ }
+
if (!ctx.arg.ltoObjPath.empty()) {
saveBuffer(buf[0].second, ctx.arg.ltoObjPath);
for (unsigned i = 1; i != maxTasks; ++i)
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 64c42eb49607d..1f52876ca5f6d 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -660,6 +660,11 @@ def lto_CGO: JJ<"lto-CGO">, MetaVarName<"<cgopt-level>">,
HelpText<"Codegen optimization level for LTO">;
def lto_partitions: JJ<"lto-partitions=">,
HelpText<"Number of LTO codegen partitions">;
+def lto_partitions_cache_dir: JJ<"lto-partitions-cache-dir=">,
+ HelpText<"Path to cache directory for regular LTO codegen partitions "
+ "(only used when --lto-partitions > 1)">;
+defm lto_partitions_cache_pruning: EEq<"lto-partitions-cache-policy",
+ "Pruning policy for the --lto-partitions-cache-dir directory">;
def lto_cs_profile_generate: FF<"lto-cs-profile-generate">,
HelpText<"Perform context sensitive PGO instrumentation">;
def lto_cs_profile_file: JJ<"lto-cs-profile-file=">,
diff --git a/lld/test/ELF/lto/lto-partitions-cache-warnings.ll b/lld/test/ELF/lto/lto-partitions-cache-warnings.ll
new file mode 100644
index 0000000000000..c5727c116de36
--- /dev/null
+++ b/lld/test/ELF/lto/lto-partitions-cache-warnings.ll
@@ -0,0 +1,57 @@
+; REQUIRES: x86
+
+; RUN: rm -rf %t && mkdir %t && cd %t
+
+; RUN: llvm-as -o %t.bc %s
+
+;; Check cache policies of the number of files.
+;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
+; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=0 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
+;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
+; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=3 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
+;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=1 %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN
+
+;; Check cache policies of the cache size.
+;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
+; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=0 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
+
+;; Get the total size of created cache files.
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=32k %t.bc -o %t3 2>&1
+; RUN: %python -c "import os, sys; size=sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')); print(size+5); print(size-5)" > %t.size.txt
+
+;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
+; RUN: echo -n "--lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=" > %t.response
+; RUN: head -1 %t.size.txt >> %t.response
+; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t @%t.response %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
+
+;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
+; RUN: echo -n "--lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=" > %t.response
+; RUN: tail -1 %t.size.txt >> %t.response
+; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t @%t.response %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN
+
+;; Check emit two warnings if pruning happens due to reach both the size and number limits.
+; RUN: ld.lld --lto-partitions-cache-dir=%t --lto-partitions=2 --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN
+
+; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
+; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
+; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+ call void @bar()
+ ret void
+}
+
+define void @bar() {
+ call void @foo()
+ ret void
+
+}
+define i32 @_start() {
+entry:
+ ret i32 0
+}
diff --git a/lld/test/ELF/lto/lto-partitions-cache.ll b/lld/test/ELF/lto/lto-partitions-cache.ll
new file mode 100644
index 0000000000000..8f82f7ba18c21
--- /dev/null
+++ b/lld/test/ELF/lto/lto-partitions-cache.ll
@@ -0,0 +1,122 @@
+; REQUIRES: x86
+; NetBSD: noatime mounts currently inhibit 'touch' from updating atime
+; UNSUPPORTED: system-netbsd
+
+; RUN: rm -rf %t && mkdir %t && cd %t
+; RUN: llvm-as -o %t.bc %s
+
+; RUN: rm -rf %t/cache && mkdir %t/cache
+; Create two files that would be removed by cache pruning due to age.
+; We should only remove files matching the pattern "llvmcache-*".
+; RUN: touch -t 197001011200 %t/cache/llvmcache-foo cache/foo
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=1h:prune_interval=0s -o out %t.bc
+
+; Two cached objects, plus a timestamp file and "foo", minus the file we removed.
+; RUN: ls %t/cache | count 4
+
+; Create a file of size 64KB.
+; RUN: %python -c "print(' ' * 65536)" > %t/cache/llvmcache-foo
+
+; This should leave the file in place.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy cache_size_bytes=128k:prune_interval=0s -o out %t.bc
+; RUN: ls %t/cache | count 5
+
+; Increase the age of llvmcache-foo, which will give it the oldest time stamp
+; so that it is processed and removed first.
+; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' %t/cache/llvmcache-foo
+
+; This should remove it.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy cache_size_bytes=32k:prune_interval=0s -o out %t.bc
+; RUN: ls %t/cache | count 4
+
+; Setting max number of files to 0 should disable the limit, not delete everything.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o out %t.bc
+; RUN: ls %t/cache | count 4
+
+; Delete everything except for the timestamp, "foo" and one cache file.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o out %t.bc
+; RUN: ls %t/cache | count 3
+
+; Check that we remove the least recently used file first.
+; RUN: rm -fr %t/cache && mkdir %t/cache
+; RUN: echo xyz > %t/cache/llvmcache-old
+; RUN: touch -t 198002011200 %t/cache/llvmcache-old
+; RUN: echo xyz > %t/cache/llvmcache-newer
+; RUN: touch -t 198002021200 %t/cache/llvmcache-newer
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s -o out %t.bc
+; RUN: ls %t/cache | FileCheck %s
+
+; CHECK-NOT: llvmcache-old
+; CHECK: llvmcache-newer
+; CHECK-NOT: llvmcache-old
+
+; RUN: rm -fr %t/cache && mkdir %t/cache
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --save-temps -o out %t.bc -M | FileCheck %s --check-prefix=MAP
+; RUN: ls out.lto.1.o out.lto.o
+
+; MAP: out.lto.o:(.text)
+; MAP: out.lto.1.o:(.text)
+; MAP: out.lto.1.o:(.text._start)
+
+;; Check that mllvm options participate in the cache key
+; RUN: rm -rf %t/cache && mkdir %t/cache
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc
+; RUN: ls %t/cache | count 3
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 5
+
+;; Adding another option resuls in 2 more cache entries
+; RUN: rm -rf cache && mkdir cache
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc
+; RUN: ls %t/cache | count 3
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 5
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ls %t/cache | count 7
+
+;; Changing order may matter - e.g. if overriding -mllvm options - so we get 2 more entries
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -max-devirt-iterations=1 -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 9
+
+;; Going back to a pre-cached order doesn't create more entries.
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ls %t/cache | count 9
+
+;; Different flag values matter
+; RUN: rm -rf cache && mkdir cache
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=2
+; RUN: ls %t/cache | count 3
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ls %t/cache | count 5
+
+;; Same flag value passed to different flags matters, and switching the order
+;; of the two flags matters.
+; RUN: rm -rf %t/cache && mkdir %t/cache
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 3
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default
+; RUN: ls %t/cache | count 5
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 5
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -emit-dwarf-unwind=default
+; RUN: ls %t/cache | count 7
+; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default -mllvm -enable-ml-inliner=default
+; RUN: ls %t/cache | count 9
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+ call void @bar()
+ ret void
+}
+
+define void @bar() {
+ call void @foo()
+ ret void
+
+}
+define i32 @_start() {
+entry:
+ ret i32 0
+}
diff --git a/lld/test/ELF/lto/cache-warnings.ll b/lld/test/ELF/lto/thinlto-cache-warnings.ll
similarity index 100%
rename from lld/test/ELF/lto/cache-warnings.ll
rename to lld/test/ELF/lto/thinlto-cache-warnings.ll
diff --git a/lld/test/ELF/lto/cache.ll b/lld/test/ELF/lto/thinlto-cache.ll
similarity index 100%
rename from lld/test/ELF/lto/cache.ll
rename to lld/test/ELF/lto/thinlto-cache.ll
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index 13b83ec6e0770..304eca7546b76 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -89,12 +89,13 @@ class LLVM_ABI DTLTO : public LTO {
/// Runs the DTLTO pipeline. This function calls the supplied AddStream
/// function to add native object files to the link.
///
- /// The Cache parameter is optional. If supplied, it will be used to cache
- /// native object files and add them to the link.
+ /// The ThinLTOCache parameter is optional. If supplied, it will be used to
+ /// cache ThinLTO backend compilations and add them to the link.
///
- /// The client will receive at most one callback (via either AddStream or
- /// Cache) for each task identifier.
- virtual Error run(AddStreamFn AddStream, FileCache Cache = {}) override;
+ /// RegularLTOCache is ignored. We always run in LTOK_UnifiedThin mode, so
+ /// distribution/caching is handled per-ThinLTO module via Cache above.
+ virtual Error run(AddStreamFn AddStream, FileCache ThinLTOCache = {},
+ FileCache RegularLTOCache = {}) override;
/// Wait for LTO cleanup. Clients may call this after run() once subsequent
/// linking work that can overlap with cleanup is complete. Cleanup may emit
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index cea66f99b99bb..29d27bc78233a 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -68,6 +68,14 @@ LLVM_ABI void thinLTOInternalizeAndPromoteInIndex(
isPrevailing,
DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr = nullptr);
+/// Computes a unique hash for the given \p Config.
+/// This hash includes the compiler revision as well.
+/// \param Conf LTO Config; only the options that affect code generation are
+/// hashed.
+/// \param Out[out] Output buffer for the hash.
+LLVM_ABI void computeLTOConfigHash(const lto::Config &Conf,
+ SmallVectorImpl<uint8_t> &Out);
+
/// Computes a unique hash for the Module considering the current list of
/// export/import and other global analysis results.
LLVM_ABI std::string computeLTOCacheKey(
@@ -432,12 +440,19 @@ class LLVM_ABI LTO {
/// Runs the LTO pipeline. This function calls the supplied AddStream
/// function to add native object files to the link.
///
- /// The Cache parameter is optional. If supplied, it will be used to cache
- /// native object files and add them to the link.
+ /// The \p ThinLTOCache parameter is optional. If supplied, it will be used to
+ /// cache native object files from ThinLTO compilation, and add them to the
+ /// link.
+ ///
+ /// The \p RegularLTOCache parameter is optional. If supplied, it will be used
+ /// to cache native object files (and add them to the link) from the
+ /// individual module partitions created when parallel codegen is enabled for
+ /// regular LTO. This is a distinct cache from \p ThinLTOCache.
///
/// The client will receive at most one callback (via either AddStream or
- /// Cache) for each task identifier.
- virtual Error run(AddStreamFn AddStream, FileCache Cache = {});
+ /// one of the FileCache) for each task identifier.
+ virtual Error run(AddStreamFn AddStream, FileCache ThinLTOCache = {},
+ FileCache RegularLTOCache = {});
/// Wait for cleanup work started by run() to finish.
///
@@ -634,7 +649,7 @@ class LLVM_ABI LTO {
addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
ArrayRef<SymbolResolution> Res);
- Error runRegularLTO(AddStreamFn AddStream);
+ Error runRegularLTO(AddStreamFn AddStream, FileCache <OPartitionsCache);
Error runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols);
diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h
index 4bb38529ec754..db4f4253339a3 100644
--- a/llvm/include/llvm/LTO/LTOBackend.h
+++ b/llvm/include/llvm/LTO/LTOBackend.h
@@ -45,6 +45,7 @@ LLVM_ABI bool opt(const Config &Conf, TargetMachine *TM, unsigned Task,
/// Runs a regular LTO backend. The regular LTO backend can also act as the
/// regular LTO phase of ThinLTO, which may need to access the combined index.
LLVM_ABI Error backend(const Config &C, AddStreamFn AddStream,
+ FileCache &PartitionsFC,
unsigned ParallelCodeGenParallelismLevel, Module &M,
ModuleSummaryIndex &CombinedIndex,
ArrayRef<StringRef> BitcodeLibFuncs);
diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
index caff198358caa..2d48569b5abd9 100644
--- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -43,6 +43,7 @@
#include "llvm/IR/Module.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
+#include "llvm/Support/CachePruning.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Error.h"
@@ -109,6 +110,87 @@ struct LTOCodeGenerator {
SaveIRBeforeOptPath = std::move(Value);
}
+ /**
+ * \defgroup Cache controlling options
+ *
+ * These entry points control the regular LTO cache used by parallel code
+ * generation (`splitModule`). The cache is intended to support incremental
+ * build, and thus needs to be persistent accross build. The client enabled
+ * the cache by supplying a path to an existing directory. The code generator
+ * will use this to store objects files that may be reused during a subsequent
+ * build. To avoid filling the disk space, a few knobs are provided:
+ * - The pruning interval limit the frequency at which the garbage collector
+ * will try to scan the cache directory to prune it from expired entries.
+ * Setting to -1 disable the pruning (default). Setting to 0 will force
+ * pruning to occur.
+ * - The pruning expiration time indicates to the garbage collector how old
+ * an entry needs to be to be removed.
+ * - Finally, the garbage collector can be instructed to prune the cache till
+ * the occupied space goes below a threshold.
+ * @{
+ */
+
+ struct CachingOptions {
+ std::string Path; // Path to the cache, empty to disable.
+ CachePruningPolicy Policy;
+ };
+
+ /// Provide a path to a directory where to store the cached files for
+ /// incremental build.
+ void setCacheDir(std::string Path) { CacheOptions.Path = std::move(Path); }
+
+ /// Cache policy: interval (seconds) between two prunes of the cache. Set to a
+ /// negative value to disable pruning. A value of 0 will force pruning to
+ /// occur.
+ void setCachePruningInterval(int Interval) {
+ if (Interval < 0)
+ CacheOptions.Policy.Interval.reset();
+ else
+ CacheOptions.Policy.Interval = std::chrono::seconds(Interval);
+ }
+
+ /// Cache policy: expiration (in seconds) for an entry.
+ /// A value of 0 will be ignored.
+ void setCacheEntryExpiration(unsigned Expiration) {
+ if (Expiration)
+ CacheOptions.Policy.Expiration = std::chrono::seconds(Expiration);
+ }
+
+ /**
+ * Sets the maximum cache size that can be persistent across build, in terms
+ * of percentage of the available space on the disk. Set to 100 to indicate
+ * no limit, 50 to indicate that the cache size will not be left over
+ * half the available space. A value over 100 will be reduced to 100, and a
+ * value of 0 will be ignored.
+ *
+ *
+ * The formula looks like:
+ * AvailableSpace = FreeSpace + ExistingCacheSize
+ * NewCacheSize = AvailableSpace * P/100
+ *
+ */
+ void setMaxCacheSizeRelativeToAvailableSpace(unsigned Percentage) {
+ if (Percentage)
+ CacheOptions.Policy.MaxSizePercentageOfAvailableSpace = Percentage;
+ }
+
+ /// Cache policy: the maximum size for the cache directory in bytes. A value
+ /// over the amount of available space on the disk will be reduced to the
+ /// amount of available space. A value of 0 will be ignored.
+ void setCacheMaxSizeBytes(uint64_t MaxSizeBytes) {
+ if (MaxSizeBytes)
+ CacheOptions.Policy.MaxSizeBytes = MaxSizeBytes;
+ }
+
+ /// Cache policy: the maximum number of files in the cache directory. A value
+ /// of 0 will be ignored.
+ void setCacheMaxSizeFiles(unsigned MaxSizeFiles) {
+ if (MaxSizeFiles)
+ CacheOptions.Policy.MaxSizeFiles = MaxSizeFiles;
+ }
+
+ /**@}*/
+
/// Restore linkage of globals
///
/// When set, the linkage of globals will be restored prior to code
@@ -247,6 +329,7 @@ struct LTOCodeGenerator {
LLVMRemarkFileHandle DiagnosticOutputFile;
std::unique_ptr<ToolOutputFile> StatsFile = nullptr;
std::string SaveIRBeforeOptPath;
+ CachingOptions CacheOptions;
lto::Config Config;
};
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index 6946abe470773..9e4e8e50d46ff 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -106,7 +106,8 @@ Error lto::DTLTO::performThinLink() {
}
// Runs the DTLTO pipeline.
-LLVM_ABI Error lto::DTLTO::run(AddStreamFn AddStream, FileCache CacheParam) {
+LLVM_ABI Error lto::DTLTO::run(AddStreamFn AddStream, FileCache CacheParam,
+ FileCache /*RegularLTOCache*/) {
scope_exit CleanUp([this]() { cleanup(); });
AddStreamFunc = AddStream;
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a8705e3d925a5..819d44cb024ce 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -132,23 +132,7 @@ extern cl::opt<bool> SupportsHotColdNew;
extern cl::opt<bool> EnableMemProfContextDisambiguation;
} // namespace llvm
-// Computes a unique hash for the Module considering the current list of
-// export/import and other global analysis results.
-// Returns the hash in its hexadecimal representation.
-std::string llvm::computeLTOCacheKey(
- const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID,
- const FunctionImporter::ImportMapTy &ImportList,
- const FunctionImporter::ExportSetTy &ExportList,
- const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
- const GVSummaryMapTy &DefinedGlobals,
- const DenseSet<GlobalValue::GUID> &CfiFunctionDefs,
- const DenseSet<GlobalValue::GUID> &CfiFunctionDecls) {
- // Compute the unique hash for this entry.
- // This is based on the current compiler version, the module itself, the
- // export list, the hash for every single module in the import list, the
- // list of ResolvedODR for the module, and the list of preserved symbols.
- SHA1 Hasher;
-
+static void hashLTOConfig(const lto::Config &Conf, SHA1 &Hasher) {
// Start with the compiler revision
Hasher.update(LLVM_VERSION_STRING);
#ifdef LLVM_REVISION
@@ -165,11 +149,6 @@ std::string llvm::computeLTOCacheKey(
support::endian::write32le(Data, I);
Hasher.update(Data);
};
- auto AddUint64 = [&](uint64_t I) {
- uint8_t Data[8];
- support::endian::write64le(Data, I);
- Hasher.update(Data);
- };
auto AddUint8 = [&](const uint8_t I) {
Hasher.update(ArrayRef<uint8_t>(&I, 1));
};
@@ -205,6 +184,66 @@ std::string llvm::computeLTOCacheKey(
AddString(Conf.DwoDir);
AddUint8(Conf.Dtlto);
+ if (!Conf.SampleProfile.empty()) {
+ auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
+ if (FileOrErr) {
+ Hasher.update(FileOrErr.get()->getBuffer());
+
+ if (!Conf.ProfileRemapping.empty()) {
+ FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
+ if (FileOrErr)
+ Hasher.update(FileOrErr.get()->getBuffer());
+ }
+ }
+ }
+}
+
+void llvm::computeLTOConfigHash(const lto::Config &Conf,
+ SmallVectorImpl<uint8_t> &Out) {
+ SHA1 Hasher;
+ hashLTOConfig(Conf, Hasher);
+ std::array<uint8_t, 20> Res = Hasher.result();
+ Out.append(Res.begin(), Res.end());
+}
+
+// Computes a unique hash for the Module considering the current list of
+// export/import and other global analysis results.
+// Returns the hash in its hexadecimal representation.
+std::string llvm::computeLTOCacheKey(
+ const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID,
+ const FunctionImporter::ImportMapTy &ImportList,
+ const FunctionImporter::ExportSetTy &ExportList,
+ const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
+ const GVSummaryMapTy &DefinedGlobals,
+ const DenseSet<GlobalValue::GUID> &CfiFunctionDefs,
+ const DenseSet<GlobalValue::GUID> &CfiFunctionDecls) {
+ // Compute the unique hash for this entry.
+ // This is based on the current compiler version, the module itself, the
+ // export list, the hash for every single module in the import list, the
+ // list of ResolvedODR for the module, and the list of preserved symbols.
+ SHA1 Hasher;
+
+ hashLTOConfig(Conf, Hasher);
+
+ // Include the parts of the LTO configuration that affect code generation.
+ auto AddString = [&](StringRef Str) {
+ Hasher.update(Str);
+ Hasher.update(ArrayRef<uint8_t>{0});
+ };
+ auto AddUnsigned = [&](unsigned I) {
+ uint8_t Data[4];
+ support::endian::write32le(Data, I);
+ Hasher.update(Data);
+ };
+ auto AddUint64 = [&](uint64_t I) {
+ uint8_t Data[8];
+ support::endian::write64le(Data, I);
+ Hasher.update(Data);
+ };
+ auto AddUint8 = [&](const uint8_t I) {
+ Hasher.update(ArrayRef<uint8_t>(&I, 1));
+ };
+
// Include the hash for the current module
auto ModHash = Index.getModuleHash(ModuleID);
Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
@@ -375,19 +414,6 @@ std::string llvm::computeLTOCacheKey(
for (auto &V : UsedCfiDecls)
AddUint64(V);
- if (!Conf.SampleProfile.empty()) {
- auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
- if (FileOrErr) {
- Hasher.update(FileOrErr.get()->getBuffer());
-
- if (!Conf.ProfileRemapping.empty()) {
- FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
- if (FileOrErr)
- Hasher.update(FileOrErr.get()->getBuffer());
- }
- }
- }
-
return toHex(Hasher.result());
}
@@ -1319,7 +1345,8 @@ Error LTO::checkPartiallySplit() {
return Error::success();
}
-Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
+Error LTO::run(AddStreamFn AddStream, FileCache ThinLTOCache,
+ FileCache RegularLTOCache) {
// Call the base class cleanup() explicitly since run() may be invoked on a
// derived LTO object.
llvm::scope_exit CleanUp([this]() { LTO::cleanup(); });
@@ -1371,11 +1398,11 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
if (SupportsHotColdNew)
ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
- Error Result = runRegularLTO(AddStream);
+ Error Result = runRegularLTO(AddStream, RegularLTOCache);
if (!Result)
// This will reset the GlobalResolutions optional once done with it to
// reduce peak memory before importing.
- Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
+ Result = runThinLTO(AddStream, ThinLTOCache, GUIDPreservedSymbols);
if (StatsFile)
PrintStatisticsJSON(StatsFile->os());
@@ -1383,7 +1410,7 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
return Result;
}
-Error LTO::runRegularLTO(AddStreamFn AddStream) {
+Error LTO::runRegularLTO(AddStreamFn AddStream, FileCache <OPartitionsCache) {
llvm::TimeTraceScope timeScope("Run regular LTO");
LLVM_DEBUG(dbgs() << "Running regular LTO\n");
@@ -1500,9 +1527,10 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
}
if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
- if (Error Err = backend(
- Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
- *RegularLTO.CombinedModule, ThinLTO.CombinedIndex, BitcodeLibFuncs))
+ if (Error Err = backend(Conf, AddStream, LTOPartitionsCache,
+ RegularLTO.ParallelCodeGenParallelismLevel,
+ *RegularLTO.CombinedModule, ThinLTO.CombinedIndex,
+ BitcodeLibFuncs))
return Err;
}
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 69bc3fdae6c57..8ada62fe3c1dd 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -36,6 +36,7 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/SHA1.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/VirtualFileSystem.h"
@@ -512,8 +513,34 @@ static void codegen(const Config &Conf, TargetMachine *TM,
report_fatal_error(std::move(Err));
}
+static std::string computeLTOPartitionCacheKey(const Config &C,
+ const SmallString<0> &BC) {
+ SHA1 Hasher;
+
+ // Hash the LTO configuration as that affects code-gen.
+ SmallVector<uint8_t, 20> ConfigHash;
+ computeLTOConfigHash(C, ConfigHash);
+ Hasher.update(ConfigHash);
+
+ // Hash the module bitcode as a whole.
+ Hasher.update(BC);
+
+ return toHex(Hasher.result());
+}
+
+static AddStreamFn getCachedAddStream(const Config &C, SmallString<0> &BC,
+ FileCache &PartitionsFC, unsigned Task,
+ StringRef ModuleName) {
+ std::string Key = computeLTOPartitionCacheKey(C, BC);
+ Expected<AddStreamFn> CacheAddStreamOrErr =
+ PartitionsFC(Task, Key, ModuleName);
+ if (Error Err = CacheAddStreamOrErr.takeError())
+ reportFatalInternalError(std::move(Err));
+ return *CacheAddStreamOrErr;
+}
+
static void splitCodeGen(const Config &C, TargetMachine *TM,
- AddStreamFn AddStream,
+ AddStreamFn AddStream, FileCache &PartitionsFC,
unsigned ParallelCodeGenParallelismLevel, Module &Mod,
const ModuleSummaryIndex &CombinedIndex) {
DefaultThreadPool CodegenThreadPool(
@@ -521,6 +548,9 @@ static void splitCodeGen(const Config &C, TargetMachine *TM,
unsigned ThreadCount = 0;
const Target *T = &TM->getTarget();
+ LLVM_DEBUG(dbgs() << "splitCodeGen for " << ParallelCodeGenParallelismLevel
+ << " partitions\n");
+
const auto HandleModulePartition =
[&](std::unique_ptr<Module> MPart) {
// We want to clone the module in a new context to multi-thread the
@@ -533,9 +563,34 @@ static void splitCodeGen(const Config &C, TargetMachine *TM,
raw_svector_ostream BCOS(BC);
WriteBitcodeToFile(*MPart, BCOS);
+ unsigned Task = ThreadCount++;
+ AddStreamFn PartitionAddStream = AddStream;
+ if (PartitionsFC.isValid()) {
+ std::string ModuleIDStr = MPart->getModuleIdentifier();
+ AddStreamFn CacheAddStream =
+ getCachedAddStream(C, BC, PartitionsFC, Task, ModuleIDStr);
+
+ // Cache hit, object was added to the stream.
+ if (!CacheAddStream) {
+ LLVM_DEBUG(dbgs() << " - splitCodeGen FileCache hit for "
+ << ModuleIDStr << "\n");
+ return;
+ }
+
+ LLVM_DEBUG(dbgs() << " - splitCodeGen FileCache missed for "
+ << ModuleIDStr << "\n");
+
+ // Cache miss, compile and add to the cache stream instead of
+ // `AddStream`.
+ PartitionAddStream = std::move(CacheAddStream);
+ } else {
+ LLVM_DEBUG(dbgs() << " - splitCodeGen FileCache not valid\n");
+ }
+
// Enqueue the task
CodegenThreadPool.async(
- [&](const SmallString<0> &BC, unsigned ThreadId) {
+ [&, PartitionAddStream](const SmallString<0> &BC,
+ unsigned ThreadId) {
LTOLLVMContext Ctx(C);
Expected<std::unique_ptr<Module>> MOrErr =
parseBitcodeFile(MemoryBufferRef(BC.str(), "ld-temp.o"), Ctx);
@@ -546,12 +601,12 @@ static void splitCodeGen(const Config &C, TargetMachine *TM,
std::unique_ptr<TargetMachine> TM =
createTargetMachine(C, T, *MPartInCtx);
- codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx,
+ codegen(C, TM.get(), PartitionAddStream, ThreadId, *MPartInCtx,
CombinedIndex);
},
// Pass BC using std::move to ensure that it get moved rather than
// copied into the thread's context.
- std::move(BC), ThreadCount++);
+ std::move(BC), Task);
};
// Try target-specific module splitting first, then fallback to the default.
@@ -593,6 +648,7 @@ Error lto::finalizeOptimizationRemarks(LLVMRemarkFileHandle DiagOutputFile) {
}
Error lto::backend(const Config &C, AddStreamFn AddStream,
+ FileCache &PartitionsFC,
unsigned ParallelCodeGenParallelismLevel, Module &Mod,
ModuleSummaryIndex &CombinedIndex,
ArrayRef<StringRef> BitcodeLibFuncs) {
@@ -614,8 +670,8 @@ Error lto::backend(const Config &C, AddStreamFn AddStream,
if (ParallelCodeGenParallelismLevel == 1) {
codegen(C, TM.get(), AddStream, 0, Mod, CombinedIndex);
} else {
- splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel, Mod,
- CombinedIndex);
+ splitCodeGen(C, TM.get(), AddStream, PartitionsFC,
+ ParallelCodeGenParallelismLevel, Mod, CombinedIndex);
}
return Error::success();
}
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index 8ae6dff4c96fe..ad11535b4a53c 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -40,6 +40,8 @@
#include "llvm/Linker/Linker.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
@@ -644,11 +646,35 @@ bool LTOCodeGenerator::compileOptimized(AddStreamFn AddStream,
ModuleSummaryIndex CombinedIndex(false);
Config.CodeGenOnly = true;
- Error Err = backend(Config, AddStream, ParallelismLevel, *MergedModule,
- CombinedIndex, /*BitcodeLibFuncs=*/{});
+
+ // If a cache directory was configured, build a
+ // FileCache to cache the codegen partitions produced when ParallelismLevel
+ // > 1.
+ FileCache PartitionsFC;
+ if (!CacheOptions.Path.empty()) {
+ Expected<FileCache> FCOrErr =
+ localCache("LTO", "lto-partition", CacheOptions.Path);
+ if (!FCOrErr) {
+ emitError(toString(FCOrErr.takeError()));
+ return false;
+ }
+ PartitionsFC = std::move(*FCOrErr);
+ }
+
+ Error Err = backend(Config, AddStream, PartitionsFC, ParallelismLevel,
+ *MergedModule, CombinedIndex, /*BitcodeLibFuncs=*/{});
assert(!Err && "unexpected code-generation failure");
(void)Err;
+ if (!CacheOptions.Path.empty()) {
+ Expected<bool> PrunedOrErr =
+ pruneCache(CacheOptions.Path, CacheOptions.Policy);
+ if (!PrunedOrErr) {
+ emitError(toString(PrunedOrErr.takeError()));
+ return false;
+ }
+ }
+
// If statistics were requested, save them to the specified file or
// print them out after codegen.
if (StatsFile)
>From 1427c524240ac3b051f12bb659b3fb88e14a30be Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Wed, 26 Aug 2026 10:10:43 +0200
Subject: [PATCH 2/2] Address comments
---
lld/ELF/Config.h | 3 +-
lld/ELF/Driver.cpp | 8 +--
lld/ELF/LTO.cpp | 31 +++-------
lld/ELF/Options.td | 7 +--
.../ELF/lto/lto-partitions-cache-warnings.ll | 57 -------------------
lld/test/ELF/lto/lto-partitions-cache.ll | 42 +++++++-------
llvm/include/llvm/DTLTO/DTLTO.h | 15 +++--
llvm/include/llvm/LTO/LTO.h | 19 +++----
llvm/lib/DTLTO/DTLTO.cpp | 2 +-
llvm/lib/LTO/LTO.cpp | 11 ++--
10 files changed, 59 insertions(+), 136 deletions(-)
delete mode 100644 lld/test/ELF/lto/lto-partitions-cache-warnings.ll
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 3d853c425ee6f..59bfc40727a45 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -489,8 +489,7 @@ struct Config {
int32_t splitStackAdjustSize;
SmallVector<uint8_t, 0> packageMetadata;
- llvm::StringRef ltoPartitionsCacheDir;
- llvm::CachePruningPolicy ltoPartitionsCachePolicy;
+ bool ltoPartitionsUsesThinLTOCache;
// The following config options do not directly correspond to any
// particular command line options.
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index bcd7cf8da95a7..10ca85e4d3404 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1496,12 +1496,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
ErrAlways(ctx) << "invalid codegen optimization level for LTO: " << ltoCgo;
ctx.arg.ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
ctx.arg.ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
- ctx.arg.ltoPartitionsCacheDir =
- args.getLastArgValue(OPT_lto_partitions_cache_dir);
- ctx.arg.ltoPartitionsCachePolicy =
- CHECK(parseCachePruningPolicy(
- args.getLastArgValue(OPT_lto_partitions_cache_pruning)),
- "--lto-partitions-cache-pruning: invalid cache policy");
+ ctx.arg.ltoPartitionsUsesThinLTOCache =
+ args.hasArg(OPT_lto_partitions_uses_thinlto_cache);
ctx.arg.ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
ctx.arg.ltoBBAddrMap =
args.hasFlag(OPT_lto_basic_block_address_map,
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index cbe3c2552dca8..7c5a43af58ebc 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -343,17 +343,6 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
cache = check(localCache("ThinLTO", "Thin", ctx.arg.thinLTOCacheDir,
createAddBufferFn(files, filenames)));
- FileCache partitionsCache;
- if (!ctx.arg.ltoPartitionsCacheDir.empty())
- partitionsCache = check(localCache(
- "LTOPartitions", "LTOPartition", ctx.arg.ltoPartitionsCacheDir,
- [&](unsigned task, const Twine &moduleName,
- std::unique_ptr<MemoryBuffer> mb) {
- // TODO: Creates a copy for a bit, is that fine?
- buf[task].second = mb->getBuffer();
- buf[task].first = moduleName.str();
- }));
-
if (!ctx.bitcodeFiles.empty())
checkError(ctx.e, ltoObj->run(
[&](size_t task, const Twine &moduleName) {
@@ -362,7 +351,7 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
std::make_unique<raw_svector_ostream>(
buf[task].second));
},
- cache, partitionsCache));
+ cache, ctx.arg.ltoPartitionsUsesThinLTOCache));
// Emit empty index files for non-indexed files but not in single-module mode.
if (ctx.arg.thinLTOModulesToCompile.empty()) {
@@ -389,18 +378,16 @@ SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
return {};
}
- if (!ctx.arg.thinLTOCacheDir.empty())
+ if (!ctx.arg.thinLTOCacheDir.empty()) {
+ // If we re-use the ThinLTO cache for --lto-partitions, add the LTO
+ // partition buffers to "files" so cache pruning can take them into account.
+ if (ctx.arg.ltoPartitionsUsesThinLTOCache) {
+ for (auto &e : buf) {
+ files.push_back(MemoryBuffer::getMemBuffer(e.second, "", false));
+ }
+ }
check(
pruneCache(ctx.arg.thinLTOCacheDir, ctx.arg.thinLTOCachePolicy, files));
-
- if (!ctx.arg.ltoPartitionsCacheDir.empty()) {
- std::vector<std::unique_ptr<MemoryBuffer>> ltoPartitionBuffers;
- for (auto &e : buf) {
- ltoPartitionBuffers.push_back(
- MemoryBuffer::getMemBuffer(e.second, "", false));
- }
- check(pruneCache(ctx.arg.ltoPartitionsCacheDir,
- ctx.arg.ltoPartitionsCachePolicy, ltoPartitionBuffers));
}
if (!ctx.arg.ltoObjPath.empty()) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 1f52876ca5f6d..42506b542d557 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -660,11 +660,8 @@ def lto_CGO: JJ<"lto-CGO">, MetaVarName<"<cgopt-level>">,
HelpText<"Codegen optimization level for LTO">;
def lto_partitions: JJ<"lto-partitions=">,
HelpText<"Number of LTO codegen partitions">;
-def lto_partitions_cache_dir: JJ<"lto-partitions-cache-dir=">,
- HelpText<"Path to cache directory for regular LTO codegen partitions "
- "(only used when --lto-partitions > 1)">;
-defm lto_partitions_cache_pruning: EEq<"lto-partitions-cache-policy",
- "Pruning policy for the --lto-partitions-cache-dir directory">;
+def lto_partitions_uses_thinlto_cache: F<"lto-partition-uses-thinlto-cache">,
+ HelpText<"Cache LTO codegen partitions using the thinLTO cache to speed up incremental builds">;
def lto_cs_profile_generate: FF<"lto-cs-profile-generate">,
HelpText<"Perform context sensitive PGO instrumentation">;
def lto_cs_profile_file: JJ<"lto-cs-profile-file=">,
diff --git a/lld/test/ELF/lto/lto-partitions-cache-warnings.ll b/lld/test/ELF/lto/lto-partitions-cache-warnings.ll
deleted file mode 100644
index c5727c116de36..0000000000000
--- a/lld/test/ELF/lto/lto-partitions-cache-warnings.ll
+++ /dev/null
@@ -1,57 +0,0 @@
-; REQUIRES: x86
-
-; RUN: rm -rf %t && mkdir %t && cd %t
-
-; RUN: llvm-as -o %t.bc %s
-
-;; Check cache policies of the number of files.
-;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
-; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=0 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
-;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
-; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=3 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
-;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=1 %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN
-
-;; Check cache policies of the cache size.
-;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
-; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=0 %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
-
-;; Get the total size of created cache files.
-; RUN: rm -rf %t && mkdir %t && cd %t
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t --lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=32k %t.bc -o %t3 2>&1
-; RUN: %python -c "import os, sys; size=sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')); print(size+5); print(size-5)" > %t.size.txt
-
-;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
-; RUN: echo -n "--lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=" > %t.response
-; RUN: head -1 %t.size.txt >> %t.response
-; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t @%t.response %t.bc -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
-
-;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
-; RUN: echo -n "--lto-partitions-cache-policy=prune_interval=0s:cache_size_bytes=" > %t.response
-; RUN: tail -1 %t.size.txt >> %t.response
-; RUN: ld.lld --verbose --lto-partitions=2 --lto-partitions-cache-dir=%t @%t.response %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN
-
-;; Check emit two warnings if pruning happens due to reach both the size and number limits.
-; RUN: ld.lld --lto-partitions-cache-dir=%t --lto-partitions=2 --lto-partitions-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t.bc -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN
-
-; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
-; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
-; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define void @foo() {
- call void @bar()
- ret void
-}
-
-define void @bar() {
- call void @foo()
- ret void
-
-}
-define i32 @_start() {
-entry:
- ret i32 0
-}
diff --git a/lld/test/ELF/lto/lto-partitions-cache.ll b/lld/test/ELF/lto/lto-partitions-cache.ll
index 8f82f7ba18c21..4a5a8a6a1fb2d 100644
--- a/lld/test/ELF/lto/lto-partitions-cache.ll
+++ b/lld/test/ELF/lto/lto-partitions-cache.ll
@@ -9,7 +9,7 @@
; Create two files that would be removed by cache pruning due to age.
; We should only remove files matching the pattern "llvmcache-*".
; RUN: touch -t 197001011200 %t/cache/llvmcache-foo cache/foo
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=1h:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy prune_after=1h:prune_interval=0s -o out %t.bc
; Two cached objects, plus a timestamp file and "foo", minus the file we removed.
; RUN: ls %t/cache | count 4
@@ -18,7 +18,7 @@
; RUN: %python -c "print(' ' * 65536)" > %t/cache/llvmcache-foo
; This should leave the file in place.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy cache_size_bytes=128k:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy cache_size_bytes=128k:prune_interval=0s -o out %t.bc
; RUN: ls %t/cache | count 5
; Increase the age of llvmcache-foo, which will give it the oldest time stamp
@@ -26,15 +26,15 @@
; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' %t/cache/llvmcache-foo
; This should remove it.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy cache_size_bytes=32k:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o out %t.bc
; RUN: ls %t/cache | count 4
; Setting max number of files to 0 should disable the limit, not delete everything.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o out %t.bc
; RUN: ls %t/cache | count 4
; Delete everything except for the timestamp, "foo" and one cache file.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o out %t.bc
; RUN: ls %t/cache | count 3
; Check that we remove the least recently used file first.
@@ -43,7 +43,7 @@
; RUN: touch -t 198002011200 %t/cache/llvmcache-old
; RUN: echo xyz > %t/cache/llvmcache-newer
; RUN: touch -t 198002021200 %t/cache/llvmcache-newer
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --lto-partitions-cache-policy prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s -o out %t.bc
; RUN: ls %t/cache | FileCheck %s
; CHECK-NOT: llvmcache-old
@@ -51,7 +51,7 @@
; CHECK-NOT: llvmcache-old
; RUN: rm -fr %t/cache && mkdir %t/cache
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache --save-temps -o out %t.bc -M | FileCheck %s --check-prefix=MAP
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache --save-temps -o out %t.bc -M | FileCheck %s --check-prefix=MAP
; RUN: ls out.lto.1.o out.lto.o
; MAP: out.lto.o:(.text)
@@ -60,47 +60,47 @@
;; Check that mllvm options participate in the cache key
; RUN: rm -rf %t/cache && mkdir %t/cache
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc
; RUN: ls %t/cache | count 3
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 5
;; Adding another option resuls in 2 more cache entries
; RUN: rm -rf cache && mkdir cache
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc
; RUN: ls %t/cache | count 3
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 5
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
; RUN: ls %t/cache | count 7
;; Changing order may matter - e.g. if overriding -mllvm options - so we get 2 more entries
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -max-devirt-iterations=1 -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -max-devirt-iterations=1 -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 9
;; Going back to a pre-cached order doesn't create more entries.
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
; RUN: ls %t/cache | count 9
;; Different flag values matter
; RUN: rm -rf cache && mkdir cache
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=2
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=2
; RUN: ls %t/cache | count 3
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -max-devirt-iterations=1
; RUN: ls %t/cache | count 5
;; Same flag value passed to different flags matters, and switching the order
;; of the two flags matters.
; RUN: rm -rf %t/cache && mkdir %t/cache
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 3
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default
; RUN: ls %t/cache | count 5
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 5
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -emit-dwarf-unwind=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -enable-ml-inliner=default -mllvm -emit-dwarf-unwind=default
; RUN: ls %t/cache | count 7
-; RUN: ld.lld --lto-partitions=2 --lto-partitions-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default -mllvm -enable-ml-inliner=default
+; RUN: ld.lld --lto-partitions=2 --lto-partition-uses-thinlto-cache --thinlto-cache-dir=%t/cache -o out %t.bc -mllvm -emit-dwarf-unwind=default -mllvm -enable-ml-inliner=default
; RUN: ls %t/cache | count 9
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h
index 304eca7546b76..0adee0bdbffb9 100644
--- a/llvm/include/llvm/DTLTO/DTLTO.h
+++ b/llvm/include/llvm/DTLTO/DTLTO.h
@@ -89,13 +89,16 @@ class LLVM_ABI DTLTO : public LTO {
/// Runs the DTLTO pipeline. This function calls the supplied AddStream
/// function to add native object files to the link.
///
- /// The ThinLTOCache parameter is optional. If supplied, it will be used to
- /// cache ThinLTO backend compilations and add them to the link.
+ /// The Cache parameter is optional. If supplied, it will be used to cache
+ /// native object files and add them to the link.
///
- /// RegularLTOCache is ignored. We always run in LTOK_UnifiedThin mode, so
- /// distribution/caching is handled per-ThinLTO module via Cache above.
- virtual Error run(AddStreamFn AddStream, FileCache ThinLTOCache = {},
- FileCache RegularLTOCache = {}) override;
+ /// If \p CacheLTOPartitions is true, \p Cache will also be used to cache the
+ /// parallel LTO codegen partitions.
+ ///
+ /// The client will receive at most one callback (via either AddStream or
+ /// Cache) for each task identifier.
+ virtual Error run(AddStreamFn AddStream, FileCache Cache = {},
+ bool CacheLTOPartitions = false) override;
/// Wait for LTO cleanup. Clients may call this after run() once subsequent
/// linking work that can overlap with cleanup is complete. Cleanup may emit
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 29d27bc78233a..bf81657596616 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -440,19 +440,16 @@ class LLVM_ABI LTO {
/// Runs the LTO pipeline. This function calls the supplied AddStream
/// function to add native object files to the link.
///
- /// The \p ThinLTOCache parameter is optional. If supplied, it will be used to
- /// cache native object files from ThinLTO compilation, and add them to the
- /// link.
+ /// The Cache parameter is optional. If supplied, it will be used to cache
+ /// native object files and add them to the link.
///
- /// The \p RegularLTOCache parameter is optional. If supplied, it will be used
- /// to cache native object files (and add them to the link) from the
- /// individual module partitions created when parallel codegen is enabled for
- /// regular LTO. This is a distinct cache from \p ThinLTOCache.
+ /// If \p CacheLTOPartitions is true, \p Cache will also be used to cache the
+ /// parallel LTO codegen partitions.
///
/// The client will receive at most one callback (via either AddStream or
- /// one of the FileCache) for each task identifier.
- virtual Error run(AddStreamFn AddStream, FileCache ThinLTOCache = {},
- FileCache RegularLTOCache = {});
+ /// Cache) for each task identifier.
+ virtual Error run(AddStreamFn AddStream, FileCache Cache = {},
+ bool CacheLTOPartitions = false);
/// Wait for cleanup work started by run() to finish.
///
@@ -649,7 +646,7 @@ class LLVM_ABI LTO {
addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
ArrayRef<SymbolResolution> Res);
- Error runRegularLTO(AddStreamFn AddStream, FileCache <OPartitionsCache);
+ Error runRegularLTO(AddStreamFn AddStream, FileCache LTOPartitionsCache);
Error runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols);
diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp
index 9e4e8e50d46ff..6bf5513f27fab 100644
--- a/llvm/lib/DTLTO/DTLTO.cpp
+++ b/llvm/lib/DTLTO/DTLTO.cpp
@@ -107,7 +107,7 @@ Error lto::DTLTO::performThinLink() {
// Runs the DTLTO pipeline.
LLVM_ABI Error lto::DTLTO::run(AddStreamFn AddStream, FileCache CacheParam,
- FileCache /*RegularLTOCache*/) {
+ bool /*CacheLTOPartitions*/) {
scope_exit CleanUp([this]() { cleanup(); });
AddStreamFunc = AddStream;
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 819d44cb024ce..3f348f686d126 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1345,8 +1345,8 @@ Error LTO::checkPartiallySplit() {
return Error::success();
}
-Error LTO::run(AddStreamFn AddStream, FileCache ThinLTOCache,
- FileCache RegularLTOCache) {
+Error LTO::run(AddStreamFn AddStream, FileCache Cache,
+ bool CacheLTOPartitions) {
// Call the base class cleanup() explicitly since run() may be invoked on a
// derived LTO object.
llvm::scope_exit CleanUp([this]() { LTO::cleanup(); });
@@ -1398,11 +1398,12 @@ Error LTO::run(AddStreamFn AddStream, FileCache ThinLTOCache,
if (SupportsHotColdNew)
ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
- Error Result = runRegularLTO(AddStream, RegularLTOCache);
+ Error Result =
+ runRegularLTO(AddStream, CacheLTOPartitions ? Cache : FileCache());
if (!Result)
// This will reset the GlobalResolutions optional once done with it to
// reduce peak memory before importing.
- Result = runThinLTO(AddStream, ThinLTOCache, GUIDPreservedSymbols);
+ Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
if (StatsFile)
PrintStatisticsJSON(StatsFile->os());
@@ -1410,7 +1411,7 @@ Error LTO::run(AddStreamFn AddStream, FileCache ThinLTOCache,
return Result;
}
-Error LTO::runRegularLTO(AddStreamFn AddStream, FileCache <OPartitionsCache) {
+Error LTO::runRegularLTO(AddStreamFn AddStream, FileCache LTOPartitionsCache) {
llvm::TimeTraceScope timeScope("Run regular LTO");
LLVM_DEBUG(dbgs() << "Running regular LTO\n");
More information about the llvm-commits
mailing list