[lld] r268404 - Produce cpio files for --reproduce.
Rafael Espindola via llvm-commits
llvm-commits at lists.llvm.org
Tue May 3 10:30:44 PDT 2016
Author: rafael
Date: Tue May 3 12:30:44 2016
New Revision: 268404
URL: http://llvm.org/viewvc/llvm-project?rev=268404&view=rev
Log:
Produce cpio files for --reproduce.
We want --reproduce to
* not rewrite scripts and thin archives
* work with absolute paths
Given that, it pretty much has to create a full directory tree. On windows that
is problematic because of the very short maximum path limit. On most cases
users can still work around it with "--repro c:\r", but that is annoying and
not viable for automated testing.
We then need to produce some form of archive with the files. The first option
that comes to mind is .a files since we already have code for writing them.
There are a few problems with them
The format has a dedicated string table, so we cannot start writing it until
all members are known.
Regular implementations don't support creating directories. We could make
llvm-ar support that, but that is probably not a good idea.
The next natural option would be tar. The problem is that to support long path
names (which is how this started) it needs a "pax extended header" making this
an annoying format to write.
The next option I looked at seems a natural fit: cpio files.
They are available on pretty much every unix, support directories and long path
names and are really easy to write. The only slightly annoying part is a
terminator, but at least gnu cpio only prints a warning if it is missing, which
is handy for crashes. This patch still makes an effort to always create it.
Added:
lld/trunk/test/ELF/reproduce-error.s
lld/trunk/test/ELF/reproduce-windows.s
Modified:
lld/trunk/ELF/Driver.cpp
lld/trunk/ELF/Driver.h
lld/trunk/ELF/DriverUtils.cpp
lld/trunk/ELF/Error.cpp
lld/trunk/ELF/Error.h
lld/trunk/ELF/InputFiles.cpp
lld/trunk/test/ELF/reproduce-linkerscript.s
lld/trunk/test/ELF/reproduce-thin-archive.s
lld/trunk/test/ELF/reproduce.s
Modified: lld/trunk/ELF/Driver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.cpp?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/Driver.cpp (original)
+++ lld/trunk/ELF/Driver.cpp Tue May 3 12:30:44 2016
@@ -108,14 +108,14 @@ void LinkerDriver::addFile(StringRef Pat
using namespace sys::fs;
if (Config->Verbose)
outs() << Path << "\n";
- if (!Config->Reproduce.empty())
- copyInputFile(Path);
Optional<MemoryBufferRef> Buffer = readFile(Path);
if (!Buffer.hasValue())
return;
MemoryBufferRef MBRef = *Buffer;
+ maybeCopyInputFile(Path, MBRef.getBuffer());
+
switch (identify_magic(MBRef.getBuffer())) {
case file_magic::unknown:
readLinkerScript(MBRef);
@@ -252,8 +252,13 @@ void LinkerDriver::main(ArrayRef<const c
readConfigs(Args);
initLLVM(Args);
- if (!Config->Reproduce.empty())
+ if (!Config->Reproduce.empty()) {
+ std::error_code EC;
+ ReproduceArchive = llvm::make_unique<raw_fd_ostream>(
+ Config->Reproduce + ".cpio", EC, fs::F_None);
+ check(EC);
createResponseFile(Args);
+ }
createFiles(Args);
checkOptions(Args);
@@ -486,6 +491,8 @@ template <class ELFT> void LinkerDriver:
for (auto *Arg : Args.filtered(OPT_wrap))
Symtab.wrap(Arg->getValue());
+ maybeCloseReproArchive();
+
// Write the result to the file.
if (Config->GcSections)
markLive<ELFT>();
Modified: lld/trunk/ELF/Driver.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.h?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/Driver.h (original)
+++ lld/trunk/ELF/Driver.h Tue May 3 12:30:44 2016
@@ -14,6 +14,7 @@
#include "lld/Core/LLVM.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/raw_ostream.h"
@@ -29,6 +30,10 @@ public:
void addLibrary(StringRef Name);
llvm::LLVMContext Context;
+ // for --reproduce
+ std::unique_ptr<llvm::raw_fd_ostream> ReproduceArchive;
+ llvm::StringSet<> IncludedFiles;
+
private:
std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef MB);
llvm::Optional<MemoryBufferRef> readFile(StringRef Path);
@@ -69,7 +74,7 @@ void printHelp(const char *Argv0);
void printVersion();
void createResponseFile(const llvm::opt::InputArgList &Args);
-void copyInputFile(StringRef Path);
+void maybeCopyInputFile(StringRef Path, StringRef Buffer);
std::string findFromSearchPaths(StringRef Path);
std::string searchLibrary(StringRef Path);
Modified: lld/trunk/ELF/DriverUtils.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/DriverUtils.cpp?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/DriverUtils.cpp (original)
+++ lld/trunk/ELF/DriverUtils.cpp Tue May 3 12:30:44 2016
@@ -113,20 +113,48 @@ static std::string relativeToRoot(String
static std::string getDestPath(StringRef Path) {
std::string Relpath = relativeToRoot(Path);
SmallString<128> Dest;
- path::append(Dest, Config->Reproduce, Relpath);
+ path::append(Dest, path::filename(Config->Reproduce), Relpath);
return Dest.str();
}
-// Copies file Src to {Config->Reproduce}/Src.
-void elf::copyInputFile(StringRef Src) {
+static void maybePrintCpioMember(StringRef Path, StringRef Data) {
+ if (Config->Reproduce.empty())
+ return;
+
+ if (!Driver->IncludedFiles.insert(Path).second)
+ return;
+
+ raw_fd_ostream &OS = *Driver->ReproduceArchive;
+ OS << "070707"; // c_magic
+
+ // The c_dev/c_ino pair should be unique according to the spec, but no one
+ // seems to care.
+ OS << "000000"; // c_dev
+ OS << "000000"; // c_ino
+
+ OS << "100664"; // c_mode: C_ISREG | rw-rw-r--
+ OS << "000000"; // c_uid
+ OS << "000000"; // c_gid
+ OS << "000001"; // c_nlink
+ OS << "000000"; // c_rdev
+ OS << "00000000000"; // c_mtime
+ OS << format("%06o", Path.size() + 1); // c_namesize
+ OS << format("%011o", Data.size()); // c_filesize
+ OS << Path << '\0'; // c_name
+ OS << Data; // c_filedata
+}
+
+// Write file Src with content Data to the archive.
+void elf::maybeCopyInputFile(StringRef Src, StringRef Data) {
std::string Dest = getDestPath(Src);
- StringRef Dir = path::parent_path(Dest);
- if (std::error_code EC = fs::create_directories(Dir)) {
- error(EC, Dir + ": can't create directory");
+ maybePrintCpioMember(Dest, Data);
+}
+
+void elf::maybeCloseReproArchive() {
+ if (!Driver->ReproduceArchive)
return;
- }
- if (std::error_code EC = fs::copy_file(Src, Dest))
- error(EC, "failed to copy file: " + Dest);
+ maybePrintCpioMember("TRAILER!!!", "");
+ Driver->ReproduceArchive.reset();
}
// Quote a given string if it contains a space character.
@@ -148,20 +176,8 @@ static std::string rewritePath(StringRef
// "ld.lld @response.txt". Used by --reproduce. This feature is
// supposed to be used by users to report an issue to LLD developers.
void elf::createResponseFile(const opt::InputArgList &Args) {
- // Create the output directory.
- if (std::error_code EC =
- fs::create_directories(Config->Reproduce, /*IgnoreExisting=*/false)) {
- error(EC, Config->Reproduce + ": can't create directory");
- return;
- }
-
- // Open "response.txt".
- SmallString<128> Path;
- path::append(Path, Config->Reproduce, "response.txt");
- std::error_code EC;
- raw_fd_ostream OS(Path, EC, fs::OpenFlags::F_None);
- check(EC);
-
+ SmallString<0> Data;
+ raw_svector_ostream OS(Data);
// Copy the command line to response.txt while rewriting paths.
for (auto *Arg : Args) {
switch (Arg->getOption().getID()) {
@@ -185,6 +201,10 @@ void elf::createResponseFile(const opt::
OS << "\n";
}
}
+
+ SmallString<128> Dest;
+ path::append(Dest, path::filename(Config->Reproduce), "response.txt");
+ maybePrintCpioMember(Dest, Data);
}
std::string elf::findFromSearchPaths(StringRef Path) {
Modified: lld/trunk/ELF/Error.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.cpp?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/Error.cpp (original)
+++ lld/trunk/ELF/Error.cpp Tue May 3 12:30:44 2016
@@ -29,6 +29,7 @@ void warning(const Twine &Msg) { llvm::e
void error(const Twine &Msg) {
*ErrorOS << Msg << "\n";
HasError = true;
+ maybeCloseReproArchive();
}
void error(std::error_code EC, const Twine &Prefix) {
@@ -38,6 +39,7 @@ void error(std::error_code EC, const Twi
void fatal(const Twine &Msg) {
llvm::errs() << Msg << "\n";
+ maybeCloseReproArchive();
exit(1);
}
Modified: lld/trunk/ELF/Error.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.h?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/Error.h (original)
+++ lld/trunk/ELF/Error.h Tue May 3 12:30:44 2016
@@ -18,6 +18,8 @@ namespace elf {
extern bool HasError;
extern llvm::raw_ostream *ErrorOS;
+void maybeCloseReproArchive();
+
void log(const Twine &Msg);
void warning(const Twine &Msg);
Modified: lld/trunk/ELF/InputFiles.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/InputFiles.cpp?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/ELF/InputFiles.cpp (original)
+++ lld/trunk/ELF/InputFiles.cpp Tue May 3 12:30:44 2016
@@ -372,12 +372,15 @@ MemoryBufferRef ArchiveFile::getMember(c
if (!Seen.insert(C.getChildOffset()).second)
return MemoryBufferRef();
- if (!Config->Reproduce.empty() && C.getParent()->isThin())
- copyInputFile(check(C.getFullName()));
+ MemoryBufferRef Ret =
+ check(C.getMemoryBufferRef(),
+ "could not get the buffer for the member defining symbol " +
+ Sym->getName());
- return check(C.getMemoryBufferRef(),
- "could not get the buffer for the member defining symbol " +
- Sym->getName());
+ if (C.getParent()->isThin())
+ maybeCopyInputFile(check(C.getFullName()), Ret.getBuffer());
+
+ return Ret;
}
template <class ELFT>
Added: lld/trunk/test/ELF/reproduce-error.s
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-error.s?rev=268404&view=auto
==============================================================================
--- lld/trunk/test/ELF/reproduce-error.s (added)
+++ lld/trunk/test/ELF/reproduce-error.s Tue May 3 12:30:44 2016
@@ -0,0 +1,15 @@
+# Extracting the cpio archive can get over the path limit on windows.
+# REQUIRES: shell
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir
+# RUN: cd %t.dir
+
+# RUN: not ld.lld --reproduce repro abc -o t 2>&1 | FileCheck %s
+# CHECK: cannot open abc: {{N|n}}o such file or directory
+
+# RUN: grep TRAILER repro.cpio
+# RUN: cpio -id < repro.cpio
+# RUN: FileCheck --check-prefix=RSP %s < repro/response.txt
+# RSP: abc
+# RSP: -o t
Modified: lld/trunk/test/ELF/reproduce-linkerscript.s
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-linkerscript.s?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/test/ELF/reproduce-linkerscript.s (original)
+++ lld/trunk/test/ELF/reproduce-linkerscript.s Tue May 3 12:30:44 2016
@@ -6,6 +6,7 @@
# RUN: echo "INPUT(\"%t.dir/build/foo.o\")" > %t.dir/build/foo.script
# RUN: cd %t.dir
# RUN: ld.lld build/foo.script -o bar --reproduce repro
+# RUN: cpio -id < repro.cpio
# RUN: diff build/foo.script repro/%:t.dir/build/foo.script
# RUN: diff build/foo.o repro/%:t.dir/build/foo.o
Modified: lld/trunk/test/ELF/reproduce-thin-archive.s
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-thin-archive.s?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/test/ELF/reproduce-thin-archive.s (original)
+++ lld/trunk/test/ELF/reproduce-thin-archive.s Tue May 3 12:30:44 2016
@@ -6,6 +6,7 @@
# RUN: cd %t.dir
# RUN: llvm-ar --format=gnu rcT foo.a foo.o
# RUN: ld.lld -m elf_x86_64 foo.a -o bar --reproduce repro
+# RUN: cpio -id < repro.cpio
# RUN: diff foo.a repro/%:t.dir/foo.a
# RUN: diff foo.o repro/%:t.dir/foo.o
Added: lld/trunk/test/ELF/reproduce-windows.s
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-windows.s?rev=268404&view=auto
==============================================================================
--- lld/trunk/test/ELF/reproduce-windows.s (added)
+++ lld/trunk/test/ELF/reproduce-windows.s Tue May 3 12:30:44 2016
@@ -0,0 +1,8 @@
+# REQUIRES: x86
+
+# Test that we can create a repro archive on windows.
+# RUN: llvm-mc %s -o %t.o -filetype=obj -triple=x86_64-pc-linux
+# RUN: ld.lld --reproduce %t.repro %t.o -o t -shared
+# RUN: cpio -t < %t.repro.cpio | FileCheck %s
+# CHECK: {{^[^/\\]*}}.repro{{/|\\}}response.txt
+# CHECK-NEXT: .repro{{/|\\}}{{.*}}.o
Modified: lld/trunk/test/ELF/reproduce.s
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce.s?rev=268404&r1=268403&r2=268404&view=diff
==============================================================================
--- lld/trunk/test/ELF/reproduce.s (original)
+++ lld/trunk/test/ELF/reproduce.s Tue May 3 12:30:44 2016
@@ -1,8 +1,6 @@
# REQUIRES: x86
-# XXX: Temporary hack to work around windows path length limitation due to
-# the build dir for llvm-clang-lld-x86_64-scei-ps4-windows10pro-fast.
-# When we directly generate an archive this won't be an issue.
+# Extracting the cpio archive can get over the path limit on windows.
# REQUIRES: shell
# RUN: rm -rf %t.dir
@@ -10,6 +8,7 @@
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build1/foo.o
# RUN: cd %t.dir
# RUN: ld.lld --hash-style=gnu build1/foo.o -o bar -shared --as-needed --reproduce repro
+# RUN: cpio -id < repro.cpio
# RUN: diff build1/foo.o repro/%:t.dir/build1/foo.o
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
@@ -24,6 +23,7 @@
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build2/foo.o
# RUN: cd %t.dir/build2/a/b/c
# RUN: ld.lld ./../../../foo.o -o bar -shared --as-needed --reproduce repro
+# RUN: cpio -id < repro.cpio
# RUN: diff %t.dir/build2/foo.o repro/%:t.dir/build2/foo.o
# RUN: echo "{ local: *; };" > ver
@@ -33,6 +33,7 @@
# RUN: ld.lld --reproduce repro2 'foo bar' -L"foo bar" -Lfile \
# RUN: --dynamic-list dyn -rpath file --script file --version-script ver \
# RUN: --dynamic-linker "some unusual/path"
+# RUN: cpio -id < repro2.cpio
# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt
# RSP2: "{{.*}}foo bar"
# RSP2-NEXT: -L "{{.*}}foo bar"
@@ -43,10 +44,6 @@
# RSP2-NEXT: --version-script {{.+}}ver
# RSP2-NEXT: --dynamic-linker "some unusual/path"
-# RUN: not ld.lld build1/foo.o -o bar -shared --as-needed --reproduce . 2>&1 \
-# RUN: | FileCheck --check-prefix=ERROR %s
-# ERROR: can't create directory
-
.globl _start
_start:
mov $60, %rax
More information about the llvm-commits
mailing list