[lld] c931ff7 - [lld-macho] Add LTO cache support
Nico Weber via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 15 09:59:24 PDT 2021
Author: Leonard Grey
Date: 2021-07-15T12:56:13-04:00
New Revision: c931ff72bde42846db4530893a94c044879c5ae9
URL: https://github.com/llvm/llvm-project/commit/c931ff72bde42846db4530893a94c044879c5ae9
DIFF: https://github.com/llvm/llvm-project/commit/c931ff72bde42846db4530893a94c044879c5ae9.diff
LOG: [lld-macho] Add LTO cache support
This adds support for the lld-only `--thinlto-cache-policy` option, as well as
implementations for ld64's `-cache_path_lto`, `-prune_interval_lto`,
`-prune_after_lto`, and `-max_relative_cache_size_lto`.
Test is adapted from lld/test/ELF/lto/cache.ll
Differential Revision: https://reviews.llvm.org/D105922
Added:
lld/test/MachO/lto-cache.ll
Modified:
lld/MachO/Config.h
lld/MachO/Driver.cpp
lld/MachO/LTO.cpp
lld/MachO/LTO.h
lld/MachO/Options.td
Removed:
################################################################################
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 51ead8bfb6cd8..fb111901dd968 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -14,6 +14,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/CachePruning.h"
#include "llvm/Support/GlobPattern.h"
#include "llvm/Support/VersionTuple.h"
#include "llvm/TextAPI/Architecture.h"
@@ -139,6 +140,8 @@ struct Configuration {
llvm::StringRef thinLTOJobs;
llvm::StringRef umbrella;
uint32_t ltoo = 2;
+ llvm::CachePruningPolicy thinLTOCachePolicy;
+ llvm::StringRef thinLTOCacheDir;
bool deadStripDylibs = false;
bool demangle = false;
bool deadStrip = false;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index fa84c3fdc418c..3150fa5e2e41d 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -195,6 +195,35 @@ getFrameworkSearchPaths(InputArgList &args,
{"/Library/Frameworks", "/System/Library/Frameworks"});
}
+static llvm::CachePruningPolicy getLTOCachePolicy(InputArgList &args) {
+ SmallString<128> ltoPolicy;
+ auto add = [<oPolicy](Twine val) {
+ if (!ltoPolicy.empty())
+ ltoPolicy += ":";
+ val.toVector(ltoPolicy);
+ };
+ for (const Arg *arg :
+ args.filtered(OPT_thinlto_cache_policy, OPT_prune_interval_lto,
+ OPT_prune_after_lto, OPT_max_relative_cache_size_lto)) {
+ switch (arg->getOption().getID()) {
+ case OPT_thinlto_cache_policy: add(arg->getValue()); break;
+ case OPT_prune_interval_lto:
+ if (!strcmp("-1", arg->getValue()))
+ add("prune_interval=87600h"); // 10 years
+ else
+ add(Twine("prune_interval=") + arg->getValue() + "s");
+ break;
+ case OPT_prune_after_lto:
+ add(Twine("prune_after=") + arg->getValue() + "s");
+ break;
+ case OPT_max_relative_cache_size_lto:
+ add(Twine("cache_size=") + arg->getValue() + "%");
+ break;
+ }
+ }
+ return CHECK(parseCachePruningPolicy(ltoPolicy), "invalid LTO cache policy");
+}
+
namespace {
struct ArchiveMember {
MemoryBufferRef mbref;
@@ -1171,6 +1200,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
config->ltoo = args::getInteger(args, OPT_lto_O, 2);
if (config->ltoo > 3)
error("--lto-O: invalid optimization level: " + Twine(config->ltoo));
+ config->thinLTOCacheDir = args.getLastArgValue(OPT_cache_path_lto);
+ config->thinLTOCachePolicy = getLTOCachePolicy(args);
config->runtimePaths = args::getStrings(args, OPT_rpath);
config->allLoad = args.hasArg(OPT_all_load);
config->archMultiple = args.hasArg(OPT_arch_multiple);
diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp
index 061cda9c1054a..366193a27eba1 100644
--- a/lld/MachO/LTO.cpp
+++ b/lld/MachO/LTO.cpp
@@ -17,6 +17,8 @@
#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Strings.h"
#include "lld/Common/TargetOptionsCommandFlags.h"
+#include "llvm/LTO/Caching.h"
+#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
@@ -97,11 +99,28 @@ void BitcodeCompiler::add(BitcodeFile &f) {
std::vector<ObjFile *> BitcodeCompiler::compile() {
unsigned maxTasks = ltoObj->getMaxTasks();
buf.resize(maxTasks);
-
- checkError(ltoObj->run([&](size_t task) {
- return std::make_unique<lto::NativeObjectStream>(
- std::make_unique<raw_svector_ostream>(buf[task]));
- }));
+ files.resize(maxTasks);
+
+ // The -cache_path_lto option specifies the path to a directory in which
+ // to cache native object files for ThinLTO incremental builds. If a path was
+ // specified, configure LTO to use it as the cache directory.
+ lto::NativeObjectCache cache;
+ if (!config->thinLTOCacheDir.empty())
+ cache = check(
+ lto::localCache(config->thinLTOCacheDir,
+ [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+ files[task] = std::move(mb);
+ }));
+
+ checkError(ltoObj->run(
+ [&](size_t task) {
+ return std::make_unique<lto::NativeObjectStream>(
+ std::make_unique<raw_svector_ostream>(buf[task]));
+ },
+ cache));
+
+ if (!config->thinLTOCacheDir.empty())
+ pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
if (config->saveTemps) {
if (!buf[0].empty())
@@ -130,6 +149,8 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
ret.push_back(make<ObjFile>(
MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, ""));
}
-
+ for (std::unique_ptr<MemoryBuffer> &file : files)
+ if (file)
+ ret.push_back(make<ObjFile>(*file, 0, ""));
return ret;
}
diff --git a/lld/MachO/LTO.h b/lld/MachO/LTO.h
index 2577374590b7f..d64016fb588c2 100644
--- a/lld/MachO/LTO.h
+++ b/lld/MachO/LTO.h
@@ -10,6 +10,7 @@
#define LLD_MACHO_LTO_H
#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/MemoryBuffer.h"
#include <memory>
#include <vector>
@@ -35,6 +36,7 @@ class BitcodeCompiler {
private:
std::unique_ptr<llvm::lto::LTO> ltoObj;
std::vector<llvm::SmallString<0>> buf;
+ std::vector<std::unique_ptr<llvm::MemoryBuffer>> files;
};
} // namespace macho
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 9b233f4b101f1..7ac2d9834f2fc 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -68,6 +68,9 @@ def lto_O: Joined<["--"], "lto-O">,
HelpText<"Set optimization level for LTO (default: 2)">,
MetaVarName<"<opt-level>">,
Group<grp_lld>;
+def thinlto_cache_policy: Joined<["--"], "thinlto-cache-policy=">,
+ HelpText<"Pruning policy for the ThinLTO cache">,
+ Group<grp_lld>;
// This is a complete Options.td compiled from Apple's ld(1) manpage
// dated 2018-03-07 and cross checked with ld64 source code in repo
@@ -915,22 +918,18 @@ def object_path_lto : Separate<["-"], "object_path_lto">,
def cache_path_lto : Separate<["-"], "cache_path_lto">,
MetaVarName<"<path>">,
HelpText<"Use <path> as a directory for the incremental LTO cache">,
- Flags<[HelpHidden]>,
Group<grp_rare>;
def prune_interval_lto : Separate<["-"], "prune_interval_lto">,
MetaVarName<"<seconds>">,
HelpText<"Prune the incremental LTO cache after <seconds> (-1 disables pruning)">,
- Flags<[HelpHidden]>,
Group<grp_rare>;
def prune_after_lto : Separate<["-"], "prune_after_lto">,
MetaVarName<"<seconds>">,
HelpText<"Remove LTO cache entries after <seconds>">,
- Flags<[HelpHidden]>,
Group<grp_rare>;
def max_relative_cache_size_lto : Separate<["-"], "max_relative_cache_size_lto">,
MetaVarName<"<percent>">,
HelpText<"Limit the incremental LTO cache growth to <percent> of free disk, space">,
- Flags<[HelpHidden]>,
Group<grp_rare>;
def page_align_data_atoms : Flag<["-"], "page_align_data_atoms">,
HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">,
diff --git a/lld/test/MachO/lto-cache.ll b/lld/test/MachO/lto-cache.ll
new file mode 100644
index 0000000000000..8f14100a90bdb
--- /dev/null
+++ b/lld/test/MachO/lto-cache.ll
@@ -0,0 +1,94 @@
+; REQUIRES: x86
+; NetBSD: noatime mounts currently inhibit 'touch' from updating atime
+; UNSUPPORTED: system-netbsd
+
+; RUN: rm -rf %t; split-file %s %t
+; RUN: opt -module-hash -module-summary %t/foo.ll -o %t/foo.o
+; RUN: opt -module-hash -module-summary %t/bar.ll -o %t/bar.o
+
+; RUN: rm -Rf %t/cache && mkdir %t/cache
+;; Create two files that would be removed by cache pruning due to age.
+;; We should only remove files matching the pattern "llvmcache-*".
+; RUN: touch -t 197001011200 %t/cache/llvmcache-baz %t/cache/baz
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN: --thinlto-cache-policy=prune_after=1h:prune_interval=0s \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+
+;; Two cached objects, plus a timestamp file and "baz", minus the file we removed.
+; RUN: ls %t/cache | count 4
+
+;; Same thing, but with `-prune_after_lto`
+; RUN: touch -t 197001011200 %t/cache/llvmcache-baz
+; RUN: %lld -cache_path_lto %t/cache -prune_after_lto 3600 -prune_interval_lto 0 \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 4
+
+;; Create a file of size 64KB.
+; RUN: %python -c "print(' ' * 65536)" > %t/cache/llvmcache-baz
+
+;; This should leave the file in place.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN: --thinlto-cache-policy=cache_size_bytes=128k:prune_interval=0s \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 5
+
+;; Increase the age of llvmcache-baz, which will give it the oldest time stamp
+;; so that it is processed and removed first.
+; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' \
+; RUN: %t/cache/llvmcache-baz
+
+;; This should remove it.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN: --thinlto-cache-policy=cache_size_bytes=32k:prune_interval=0s \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 4
+
+;; Delete everything except for the timestamp, "baz" and one cache file.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN: --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 3
+
+;; Check that we remove the least recently used file first.
+; RUN: rm -fr %t/cache
+; RUN: mkdir %t/cache
+; RUN: echo xyz > %t/cache/llvmcache-old
+; RUN: touch -t 198002011200 %t/cache/llvmcache-old
+; RUN: echo xyz > %t/cache/llvmcache-newer
+; RUN: touch -t 198002021200 %t/cache/llvmcache-newer
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN: --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | FileCheck %s
+
+;; Check that `-max_relative_cache_size_lto` is a legal argument.
+; RUN: %lld -cache_path_lto %t/cache -max_relative_cache_size_lto 10 \
+; RUN: -o %t/test %t/foo.o %t/bar.o
+
+; CHECK-NOT: llvmcache-old
+; CHECK: llvmcache-newer
+; CHECK-NOT: llvmcache-old
+
+;--- foo.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @globalfunc() #0 {
+entry:
+ ret void
+}
+
+
+;--- bar.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @main() {
+entry:
+ call void (...) @globalfunc()
+ ret i32 0
+}
+
+declare void @globalfunc(...)
More information about the llvm-commits
mailing list