[lld] 83e60f5 - [lld/mac] Add --reproduce option
Nico Weber via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 30 05:40:31 PST 2020
Author: Nico Weber
Date: 2020-11-30T08:40:21-05:00
New Revision: 83e60f5a554d2f531b17875e835eab0cbb487211
URL: https://github.com/llvm/llvm-project/commit/83e60f5a554d2f531b17875e835eab0cbb487211
DIFF: https://github.com/llvm/llvm-project/commit/83e60f5a554d2f531b17875e835eab0cbb487211.diff
LOG: [lld/mac] Add --reproduce option
This adds support for ld.lld's --reproduce / lld-link's /reproduce:
flag to the MachO port. This flag can be added to a link command
to make the link write a tar file containing all inputs to the link
and a response file containing the link command. This can be used
to reproduce the link on another machine, which is useful for sharing
bug report inputs or performance test loads.
Since the linker is usually called through the clang driver and
adding linker flags can be a bit cumbersome, setting the env var
`LLD_REPRODUCE=foo.tar` triggers the feature as well.
The file response.txt in the archive can be used with
`ld64.lld.darwinnew $(cat response.txt)` as long as the contents are
smaller than the command-line limit, or with `ld64.lld.darwinnew
@response.txt` once D92149 is in.
The support in this patch is sufficient to create a tar file for
Chromium's base_unittests that can link after unpacking on a different
machine.
Differential Revision: https://reviews.llvm.org/D92274
Added:
lld/test/MachO/reproduce.s
Modified:
lld/Common/Reproduce.cpp
lld/ELF/DriverUtils.cpp
lld/MachO/Driver.cpp
lld/MachO/Driver.h
lld/MachO/DriverUtils.cpp
lld/MachO/InputFiles.cpp
lld/MachO/InputFiles.h
lld/MachO/Options.td
Removed:
################################################################################
diff --git a/lld/Common/Reproduce.cpp b/lld/Common/Reproduce.cpp
index 00309f58b93f..017f53df05a4 100644
--- a/lld/Common/Reproduce.cpp
+++ b/lld/Common/Reproduce.cpp
@@ -54,7 +54,12 @@ std::string lld::toString(const opt::Arg &arg) {
std::string k = std::string(arg.getSpelling());
if (arg.getNumValues() == 0)
return k;
- std::string v = quote(arg.getValue());
+ std::string v;
+ for (size_t i = 0; i < arg.getNumValues(); ++i) {
+ if (i > 0)
+ v.push_back(' ');
+ v += quote(arg.getValue(i));
+ }
if (arg.getOption().getRenderStyle() == opt::Option::RenderJoinedStyle)
return k + v;
return k + " " + v;
diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp
index 8ce9853c57af..2efd92a3df76 100644
--- a/lld/ELF/DriverUtils.cpp
+++ b/lld/ELF/DriverUtils.cpp
@@ -184,7 +184,7 @@ std::string elf::createResponseFile(const opt::InputArgList &args) {
// fail because the archive we are creating doesn't contain empty
// directories for the output path (-o doesn't create directories).
// Strip directories to prevent the issue.
- os << "-o " << quote(sys::path::filename(arg->getValue())) << "\n";
+ os << "-o " << quote(path::filename(arg->getValue())) << "\n";
break;
case OPT_dynamic_list:
case OPT_library_path:
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 0f5da218e80d..3de92f510b30 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -37,6 +37,7 @@
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
#include "llvm/Support/TargetSelect.h"
#include <algorithm>
@@ -548,6 +549,12 @@ static void warnIfUnimplementedOption(const opt::Option &opt) {
}
}
+static const char *getReproduceOption(opt::InputArgList &args) {
+ if (auto *arg = args.getLastArg(OPT_reproduce))
+ return arg->getValue();
+ return getenv("LLD_REPRODUCE");
+}
+
bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
raw_ostream &stdoutOS, raw_ostream &stderrOS) {
lld::stdoutOS = &stdoutOS;
@@ -569,6 +576,20 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
return true;
}
+ if (const char *path = getReproduceOption(args)) {
+ // Note that --reproduce is a debug option so you can ignore it
+ // if you are trying to understand the whole picture of the code.
+ Expected<std::unique_ptr<TarWriter>> errOrWriter =
+ TarWriter::create(path, path::stem(path));
+ if (errOrWriter) {
+ tar = std::move(*errOrWriter);
+ tar->append("response.txt", createResponseFile(args));
+ tar->append("version.txt", getLLDVersion() + "\n");
+ } else {
+ error("--reproduce: " + toString(errOrWriter.takeError()));
+ }
+ }
+
config = make<Configuration>();
symtab = make<SymbolTable>();
target = createTargetInfo(args);
diff --git a/lld/MachO/Driver.h b/lld/MachO/Driver.h
index 9db628c86de2..c06d2f9d1b3d 100644
--- a/lld/MachO/Driver.h
+++ b/lld/MachO/Driver.h
@@ -35,6 +35,8 @@ enum {
#undef OPTION
};
+std::string createResponseFile(const llvm::opt::InputArgList &args);
+
// Check for both libfoo.dylib and libfoo.tbd (in that order).
llvm::Optional<std::string> resolveDylibPath(llvm::StringRef path);
diff --git a/lld/MachO/DriverUtils.cpp b/lld/MachO/DriverUtils.cpp
index d17a8215f6a3..90f1dd323f02 100644
--- a/lld/MachO/DriverUtils.cpp
+++ b/lld/MachO/DriverUtils.cpp
@@ -9,8 +9,10 @@
#include "Driver.h"
#include "InputFiles.h"
+#include "lld/Common/Args.h"
#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
@@ -96,6 +98,55 @@ void MachOOptTable::printHelp(const char *argv0, bool showHidden) const {
lld::outs() << "\n";
}
+static std::string rewritePath(StringRef s) {
+ if (fs::exists(s))
+ return relativeToRoot(s);
+ return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
+std::string macho::createResponseFile(const opt::InputArgList &args) {
+ SmallString<0> data;
+ raw_svector_ostream os(data);
+
+ // Copy the command line to the output while rewriting paths.
+ for (auto *arg : args) {
+ switch (arg->getOption().getID()) {
+ case OPT_reproduce:
+ break;
+ case OPT_INPUT:
+ os << quote(rewritePath(arg->getValue())) << "\n";
+ break;
+ case OPT_o:
+ os << "-o " << quote(path::filename(arg->getValue())) << "\n";
+ break;
+ case OPT_filelist:
+ if (Optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
+ for (StringRef path : args::getLines(*buffer))
+ os << quote(rewritePath(path)) << "\n";
+ break;
+ case OPT_force_load:
+ case OPT_rpath:
+ case OPT_syslibroot:
+ case OPT_F:
+ case OPT_L:
+ case OPT_order_file:
+ os << arg->getSpelling() << " " << quote(rewritePath(arg->getValue()))
+ << "\n";
+ break;
+ case OPT_sectcreate:
+ os << arg->getSpelling() << " " << quote(arg->getValue(0)) << " "
+ << quote(arg->getValue(1)) << " "
+ << quote(rewritePath(arg->getValue(2))) << "\n";
+ break;
+ default:
+ os << toString(*arg) << "\n";
+ }
+ }
+ return std::string(data.str());
+}
+
Optional<std::string> macho::resolveDylibPath(StringRef path) {
// TODO: if a tbd and dylib are both present, we should check to make sure
// they are consistent.
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index c733878ec1fe..2f65951a49c7 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -56,12 +56,14 @@
#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
#include "llvm/ADT/iterator.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
using namespace llvm;
using namespace llvm::MachO;
@@ -71,6 +73,7 @@ using namespace lld;
using namespace lld::macho;
std::vector<InputFile *> macho::inputFiles;
+std::unique_ptr<TarWriter> macho::tar;
// Open a given file path and return it as a memory-mapped file.
Optional<MemoryBufferRef> macho::readFile(StringRef path) {
@@ -88,8 +91,11 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) {
// If this is a regular non-fat file, return it.
const char *buf = mbref.getBufferStart();
auto *hdr = reinterpret_cast<const MachO::fat_header *>(buf);
- if (read32be(&hdr->magic) != MachO::FAT_MAGIC)
+ if (read32be(&hdr->magic) != MachO::FAT_MAGIC) {
+ if (tar)
+ tar->append(relativeToRoot(path), mbref.getBuffer());
return mbref;
+ }
// Object files and archive files may be fat files, which contains
// multiple real files for
diff erent CPU ISAs. Here, we search for a
@@ -112,6 +118,8 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) {
uint32_t size = read32be(&arch[i].size);
if (offset + size > mbref.getBufferSize())
error(path + ": slice extends beyond end of file");
+ if (tar)
+ tar->append(relativeToRoot(path), mbref.getBuffer());
return MemoryBufferRef(StringRef(buf + offset, size), path.copy(bAlloc));
}
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
index 0cb7a10aa2bb..a1405aa66ea2 100644
--- a/lld/MachO/InputFiles.h
+++ b/lld/MachO/InputFiles.h
@@ -27,6 +27,7 @@ namespace llvm {
namespace lto {
class InputFile;
} // namespace lto
+class TarWriter;
} // namespace llvm
namespace lld {
@@ -36,6 +37,10 @@ class InputSection;
class Symbol;
struct Reloc;
+// If --reproduce option is given, all input files are written
+// to this tar archive.
+extern std::unique_ptr<llvm::TarWriter> tar;
+
// If .subsections_via_symbols is set, each InputSection will be split along
// symbol boundaries. The keys of a SubsectionMap represent the offsets of
// each subsection from the start of the original pre-split InputSection.
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 4aa999826fb6..f36820b22b17 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -14,6 +14,10 @@ def no_color_diagnostics: Flag<["--"], "no-color-diagnostics">,
def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
HelpText<"Use colors in diagnostics (default: auto)">,
MetaVarName<"[auto,always,never]">;
+def reproduce: Separate<["--"], "reproduce">;
+def reproduce_eq: Joined<["--"], "reproduce=">,
+ Alias<!cast<Separate>(reproduce)>,
+ HelpText<"Write tar file containing inputs and command to reproduce link">;
// This is a complete Options.td compiled from Apple's ld(1) manpage
diff --git a/lld/test/MachO/reproduce.s b/lld/test/MachO/reproduce.s
new file mode 100644
index 000000000000..19055a035e50
--- /dev/null
+++ b/lld/test/MachO/reproduce.s
@@ -0,0 +1,37 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir/build1
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build1/foo.o
+# RUN: cd %t.dir
+# RUN: %lld -platform_version macos 10.10.0 11.0 build1/foo.o -o bar --reproduce repro1.tar
+# RUN: tar xOf repro1.tar repro1/%:t.dir/build1/foo.o > build1-foo.o
+# RUN: cmp build1/foo.o build1-foo.o
+
+# RUN: tar xf repro1.tar repro1/response.txt repro1/version.txt
+# RUN: FileCheck %s --check-prefix=RSP1 < repro1/response.txt
+# RSP1: {{^}}-platform_version macos 10.10.0 11.0{{$}}
+# RSP1-NOT: {{^}}repro1{{[/\\]}}
+# RSP1-NEXT: {{[/\\]}}foo.o
+# RSP1-NEXT: -o bar
+# RSP1-NOT: --reproduce
+
+# RUN: FileCheck %s --check-prefix=VERSION < repro1/version.txt
+# VERSION: LLD
+
+# RUN: mkdir -p %t.dir/build2/a/b/c
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build2/foo.o
+# RUN: cd %t.dir/build2/a/b/c
+# RUN: echo ./../../../foo.o > %t.dir/build2/filelist
+# RUN: env LLD_REPRODUCE=repro2.tar %lld -filelist %t.dir/build2/filelist -o /dev/null
+# RUN: tar xOf repro2.tar repro2/%:t.dir/build2/foo.o > build2-foo.o
+# RUN: cmp %t.dir/build2/foo.o build2-foo.o
+
+# RUN: tar xf repro2.tar repro2/response.txt repro2/version.txt
+# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt
+# RSP2-NOT: {{^}}repro2{{[/\\]}}
+# RSP2: {{[/\\]}}foo.o
+
+.globl _main
+_main:
+ ret
More information about the llvm-commits
mailing list