<div dir="ltr"><br><div class="gmail_extra"><br><div class="gmail_quote">On Tue, May 3, 2016 at 10:30 AM, Rafael Espindola via llvm-commits <span dir="ltr"><<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left-width:1px;border-left-style:solid;border-left-color:rgb(204,204,204);padding-left:1ex">Author: rafael<br>
Date: Tue May 3 12:30:44 2016<br>
New Revision: 268404<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=268404&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=268404&view=rev</a><br>
Log:<br>
Produce cpio files for --reproduce.<br>
<br>
We want --reproduce to<br>
<br>
* not rewrite scripts and thin archives<br>
* work with absolute paths<br>
<br>
Given that, it pretty much has to create a full directory tree. On windows that<br>
is problematic because of the very short maximum path limit. On most cases<br>
users can still work around it with "--repro c:\r", but that is annoying and<br>
not viable for automated testing.<br>
<br>
We then need to produce some form of archive with the files. The first option<br>
that comes to mind is .a files since we already have code for writing them.<br>
There are a few problems with them<br>
<br>
The format has a dedicated string table, so we cannot start writing it until<br>
all members are known.<br>
Regular implementations don't support creating directories. We could make<br>
llvm-ar support that, but that is probably not a good idea.<br>
The next natural option would be tar. The problem is that to support long path<br>
names (which is how this started) it needs a "pax extended header" making this<br>
an annoying format to write.<br>
<br>
The next option I looked at seems a natural fit: cpio files.<br>
<br>
They are available on pretty much every unix, support directories and long path<br>
names and are really easy to write. The only slightly annoying part is a<br>
terminator, but at least gnu cpio only prints a warning if it is missing, which<br>
is handy for crashes. This patch still makes an effort to always create it.<br>
<br>
Added:<br>
lld/trunk/test/ELF/reproduce-error.s<br>
lld/trunk/test/ELF/reproduce-windows.s<br>
Modified:<br>
lld/trunk/ELF/Driver.cpp<br>
lld/trunk/ELF/Driver.h<br>
lld/trunk/ELF/DriverUtils.cpp<br>
lld/trunk/ELF/Error.cpp<br>
lld/trunk/ELF/Error.h<br>
lld/trunk/ELF/InputFiles.cpp<br>
lld/trunk/test/ELF/reproduce-linkerscript.s<br>
lld/trunk/test/ELF/reproduce-thin-archive.s<br>
lld/trunk/test/ELF/reproduce.s<br>
<br>
Modified: lld/trunk/ELF/Driver.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.cpp?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.cpp?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/Driver.cpp (original)<br>
+++ lld/trunk/ELF/Driver.cpp Tue May 3 12:30:44 2016<br>
@@ -108,14 +108,14 @@ void LinkerDriver::addFile(StringRef Pat<br>
using namespace sys::fs;<br>
if (Config->Verbose)<br>
outs() << Path << "\n";<br>
- if (!Config->Reproduce.empty())<br>
- copyInputFile(Path);<br>
<br>
Optional<MemoryBufferRef> Buffer = readFile(Path);<br>
if (!Buffer.hasValue())<br>
return;<br>
MemoryBufferRef MBRef = *Buffer;<br>
<br>
+ maybeCopyInputFile(Path, MBRef.getBuffer());<br>
+<br>
switch (identify_magic(MBRef.getBuffer())) {<br>
case file_magic::unknown:<br>
readLinkerScript(MBRef);<br>
@@ -252,8 +252,13 @@ void LinkerDriver::main(ArrayRef<const c<br>
readConfigs(Args);<br>
initLLVM(Args);<br>
<br>
- if (!Config->Reproduce.empty())<br>
+ if (!Config->Reproduce.empty()) {<br>
+ std::error_code EC;<br>
+ ReproduceArchive = llvm::make_unique<raw_fd_ostream>(<br>
+ Config->Reproduce + ".cpio", EC, fs::F_None);<br>
+ check(EC);<br>
createResponseFile(Args);<br>
+ }<br>
<br>
createFiles(Args);<br>
checkOptions(Args);<br>
@@ -486,6 +491,8 @@ template <class ELFT> void LinkerDriver:<br>
for (auto *Arg : Args.filtered(OPT_wrap))<br>
Symtab.wrap(Arg->getValue());<br>
<br>
+ maybeCloseReproArchive();<br>
+<br>
// Write the result to the file.<br>
if (Config->GcSections)<br>
markLive<ELFT>();<br>
<br>
Modified: lld/trunk/ELF/Driver.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.h?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.h?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/Driver.h (original)<br>
+++ lld/trunk/ELF/Driver.h Tue May 3 12:30:44 2016<br>
@@ -14,6 +14,7 @@<br>
#include "lld/Core/LLVM.h"<br>
#include "llvm/ADT/Optional.h"<br>
#include "llvm/ADT/StringRef.h"<br>
+#include "llvm/ADT/StringSet.h"<br>
#include "llvm/Option/ArgList.h"<br>
#include "llvm/Support/raw_ostream.h"<br>
<br>
@@ -29,6 +30,10 @@ public:<br>
void addLibrary(StringRef Name);<br>
llvm::LLVMContext Context;<br>
<br>
+ // for --reproduce<br>
+ std::unique_ptr<llvm::raw_fd_ostream> ReproduceArchive;<br>
+ llvm::StringSet<> IncludedFiles;<br>
+<br>
private:<br>
std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef MB);<br>
llvm::Optional<MemoryBufferRef> readFile(StringRef Path);<br>
@@ -69,7 +74,7 @@ void printHelp(const char *Argv0);<br>
void printVersion();<br>
<br>
void createResponseFile(const llvm::opt::InputArgList &Args);<br>
-void copyInputFile(StringRef Path);<br>
+void maybeCopyInputFile(StringRef Path, StringRef Buffer);<br>
<br>
std::string findFromSearchPaths(StringRef Path);<br>
std::string searchLibrary(StringRef Path);<br>
<br>
Modified: lld/trunk/ELF/DriverUtils.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/DriverUtils.cpp?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/DriverUtils.cpp?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/DriverUtils.cpp (original)<br>
+++ lld/trunk/ELF/DriverUtils.cpp Tue May 3 12:30:44 2016<br>
@@ -113,20 +113,48 @@ static std::string relativeToRoot(String<br>
static std::string getDestPath(StringRef Path) {<br>
std::string Relpath = relativeToRoot(Path);<br>
SmallString<128> Dest;<br>
- path::append(Dest, Config->Reproduce, Relpath);<br>
+ path::append(Dest, path::filename(Config->Reproduce), Relpath);<br>
return Dest.str();<br>
}<br>
<br>
-// Copies file Src to {Config->Reproduce}/Src.<br>
-void elf::copyInputFile(StringRef Src) {<br>
+static void maybePrintCpioMember(StringRef Path, StringRef Data) {<br>
+ if (Config->Reproduce.empty())<br>
+ return;<br>
+<br>
+ if (!Driver->IncludedFiles.insert(Path).second)<br>
+ return;<br>
+<br>
+ raw_fd_ostream &OS = *Driver->ReproduceArchive;<br>
+ OS << "070707"; // c_magic<br>
+<br>
+ // The c_dev/c_ino pair should be unique according to the spec, but no one<br>
+ // seems to care.<br>
+ OS << "000000"; // c_dev<br>
+ OS << "000000"; // c_ino<br>
+<br>
+ OS << "100664"; // c_mode: C_ISREG | rw-rw-r--<br>
+ OS << "000000"; // c_uid<br>
+ OS << "000000"; // c_gid<br>
+ OS << "000001"; // c_nlink<br>
+ OS << "000000"; // c_rdev<br>
+ OS << "00000000000"; // c_mtime<br>
+ OS << format("%06o", Path.size() + 1); // c_namesize<br>
+ OS << format("%011o", Data.size()); // c_filesize<br>
+ OS << Path << '\0'; // c_name<br>
+ OS << Data; // c_filedata<br>
+}<br>
+<br>
+// Write file Src with content Data to the archive.<br>
+void elf::maybeCopyInputFile(StringRef Src, StringRef Data) {<br>
std::string Dest = getDestPath(Src);<br>
- StringRef Dir = path::parent_path(Dest);<br>
- if (std::error_code EC = fs::create_directories(Dir)) {<br>
- error(EC, Dir + ": can't create directory");<br>
+ maybePrintCpioMember(Dest, Data);<br>
+}<br>
+<br>
+void elf::maybeCloseReproArchive() {<br>
+ if (!Driver->ReproduceArchive)<br>
return;<br>
- }<br>
- if (std::error_code EC = fs::copy_file(Src, Dest))<br>
- error(EC, "failed to copy file: " + Dest);<br>
+ maybePrintCpioMember("TRAILER!!!", "");<br>
+ Driver->ReproduceArchive.reset();<br>
}<br>
<br>
// Quote a given string if it contains a space character.<br>
@@ -148,20 +176,8 @@ static std::string rewritePath(StringRef<br>
// "ld.lld @response.txt". Used by --reproduce. This feature is<br>
// supposed to be used by users to report an issue to LLD developers.<br>
void elf::createResponseFile(const opt::InputArgList &Args) {<br>
- // Create the output directory.<br>
- if (std::error_code EC =<br>
- fs::create_directories(Config->Reproduce, /*IgnoreExisting=*/false)) {<br>
- error(EC, Config->Reproduce + ": can't create directory");<br>
- return;<br>
- }<br>
-<br>
- // Open "response.txt".<br>
- SmallString<128> Path;<br>
- path::append(Path, Config->Reproduce, "response.txt");<br>
- std::error_code EC;<br>
- raw_fd_ostream OS(Path, EC, fs::OpenFlags::F_None);<br>
- check(EC);<br>
-<br>
+ SmallString<0> Data;<br>
+ raw_svector_ostream OS(Data);<br>
// Copy the command line to response.txt while rewriting paths.<br>
for (auto *Arg : Args) {<br>
switch (Arg->getOption().getID()) {<br>
@@ -185,6 +201,10 @@ void elf::createResponseFile(const opt::<br>
OS << "\n";<br>
}<br>
}<br>
+<br>
+ SmallString<128> Dest;<br>
+ path::append(Dest, path::filename(Config->Reproduce), "response.txt");<br>
+ maybePrintCpioMember(Dest, Data);<br>
}<br>
<br>
std::string elf::findFromSearchPaths(StringRef Path) {<br>
<br>
Modified: lld/trunk/ELF/Error.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.cpp?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.cpp?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/Error.cpp (original)<br>
+++ lld/trunk/ELF/Error.cpp Tue May 3 12:30:44 2016<br>
@@ -29,6 +29,7 @@ void warning(const Twine &Msg) { llvm::e<br>
void error(const Twine &Msg) {<br>
*ErrorOS << Msg << "\n";<br>
HasError = true;<br>
+ maybeCloseReproArchive();<br>
}<br>
<br>
void error(std::error_code EC, const Twine &Prefix) {<br>
@@ -38,6 +39,7 @@ void error(std::error_code EC, const Twi<br>
<br>
void fatal(const Twine &Msg) {<br>
llvm::errs() << Msg << "\n";<br>
+ maybeCloseReproArchive();<br>
exit(1);<br>
}<br>
<br>
<br>
Modified: lld/trunk/ELF/Error.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.h?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.h?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/Error.h (original)<br>
+++ lld/trunk/ELF/Error.h Tue May 3 12:30:44 2016<br>
@@ -18,6 +18,8 @@ namespace elf {<br>
extern bool HasError;<br>
extern llvm::raw_ostream *ErrorOS;<br>
<br>
+void maybeCloseReproArchive();<br>
+<br>
void log(const Twine &Msg);<br>
void warning(const Twine &Msg);<br>
<br>
<br>
Modified: lld/trunk/ELF/InputFiles.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/InputFiles.cpp?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/InputFiles.cpp?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/ELF/InputFiles.cpp (original)<br>
+++ lld/trunk/ELF/InputFiles.cpp Tue May 3 12:30:44 2016<br>
@@ -372,12 +372,15 @@ MemoryBufferRef ArchiveFile::getMember(c<br>
if (!Seen.insert(C.getChildOffset()).second)<br>
return MemoryBufferRef();<br>
<br>
- if (!Config->Reproduce.empty() && C.getParent()->isThin())<br>
- copyInputFile(check(C.getFullName()));<br>
+ MemoryBufferRef Ret =<br>
+ check(C.getMemoryBufferRef(),<br>
+ "could not get the buffer for the member defining symbol " +<br>
+ Sym->getName());<br>
<br>
- return check(C.getMemoryBufferRef(),<br>
- "could not get the buffer for the member defining symbol " +<br>
- Sym->getName());<br>
+ if (C.getParent()->isThin())<br>
+ maybeCopyInputFile(check(C.getFullName()), Ret.getBuffer());<br>
+<br>
+ return Ret;<br>
}<br>
<br>
template <class ELFT><br>
<br>
Added: lld/trunk/test/ELF/reproduce-error.s<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-error.s?rev=268404&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-error.s?rev=268404&view=auto</a><br>
==============================================================================<br>
--- lld/trunk/test/ELF/reproduce-error.s (added)<br>
+++ lld/trunk/test/ELF/reproduce-error.s Tue May 3 12:30:44 2016<br>
@@ -0,0 +1,15 @@<br>
+# Extracting the cpio archive can get over the path limit on windows.<br>
+# REQUIRES: shell<br>
+<br>
+# RUN: rm -rf %t.dir<br>
+# RUN: mkdir -p %t.dir<br>
+# RUN: cd %t.dir<br>
+<br>
+# RUN: not ld.lld --reproduce repro abc -o t 2>&1 | FileCheck %s<br>
+# CHECK: cannot open abc: {{N|n}}o such file or directory<br>
+<br>
+# RUN: grep TRAILER repro.cpio<br>
+# RUN: cpio -id < repro.cpio<br>
+# RUN: FileCheck --check-prefix=RSP %s < repro/response.txt<br>
+# RSP: abc<br>
+# RSP: -o t<br>
<br>
Modified: lld/trunk/test/ELF/reproduce-linkerscript.s<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-linkerscript.s?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-linkerscript.s?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/test/ELF/reproduce-linkerscript.s (original)<br>
+++ lld/trunk/test/ELF/reproduce-linkerscript.s Tue May 3 12:30:44 2016<br>
@@ -6,6 +6,7 @@<br>
# RUN: echo "INPUT(\"%t.dir/build/foo.o\")" > %t.dir/build/foo.script<br>
# RUN: cd %t.dir<br>
# RUN: ld.lld build/foo.script -o bar --reproduce repro<br>
+# RUN: cpio -id < repro.cpio<br>
# RUN: diff build/foo.script repro/%:t.dir/build/foo.script<br>
# RUN: diff build/foo.o repro/%:t.dir/build/foo.o<br>
<br>
<br>
Modified: lld/trunk/test/ELF/reproduce-thin-archive.s<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-thin-archive.s?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-thin-archive.s?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/test/ELF/reproduce-thin-archive.s (original)<br>
+++ lld/trunk/test/ELF/reproduce-thin-archive.s Tue May 3 12:30:44 2016<br>
@@ -6,6 +6,7 @@<br>
# RUN: cd %t.dir<br>
# RUN: llvm-ar --format=gnu rcT foo.a foo.o<br>
# RUN: ld.lld -m elf_x86_64 foo.a -o bar --reproduce repro<br>
+# RUN: cpio -id < repro.cpio<br>
# RUN: diff foo.a repro/%:t.dir/foo.a<br>
# RUN: diff foo.o repro/%:t.dir/foo.o<br>
<br>
<br>
Added: lld/trunk/test/ELF/reproduce-windows.s<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-windows.s?rev=268404&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce-windows.s?rev=268404&view=auto</a><br>
==============================================================================<br>
--- lld/trunk/test/ELF/reproduce-windows.s (added)<br>
+++ lld/trunk/test/ELF/reproduce-windows.s Tue May 3 12:30:44 2016<br>
@@ -0,0 +1,8 @@<br>
+# REQUIRES: x86<br>
+<br>
+# Test that we can create a repro archive on windows.<br>
+# RUN: llvm-mc %s -o %t.o -filetype=obj -triple=x86_64-pc-linux<br>
+# RUN: ld.lld --reproduce %t.repro %t.o -o t -shared<br>
+# RUN: cpio -t < %t.repro.cpio | FileCheck %s<br>
+# CHECK: {{^[^/\\]*}}.repro{{/|\\}}response.txt<br>
+# CHECK-NEXT: .repro{{/|\\}}{{.*}}.o<br>
<br>
Modified: lld/trunk/test/ELF/reproduce.s<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce.s?rev=268404&r1=268403&r2=268404&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/lld/trunk/test/ELF/reproduce.s?rev=268404&r1=268403&r2=268404&view=diff</a><br>
==============================================================================<br>
--- lld/trunk/test/ELF/reproduce.s (original)<br>
+++ lld/trunk/test/ELF/reproduce.s Tue May 3 12:30:44 2016<br>
@@ -1,8 +1,6 @@<br>
# REQUIRES: x86<br>
<br>
-# XXX: Temporary hack to work around windows path length limitation due to<br>
-# the build dir for llvm-clang-lld-x86_64-scei-ps4-windows10pro-fast.<br>
-# When we directly generate an archive this won't be an issue.<br>
+# Extracting the cpio archive can get over the path limit on windows.<br>
# REQUIRES: shell<br></blockquote><div><br></div><div>
<p class="">In case it wasn't clear, the FIXME (well, XXX) was about the `REQUIRES: shell`. Without removing the `REQUIRES: shell` it doesn't really make sense to remove the FIXME.</p><p class="">Can we segregate the archive extraction into a separate test?</p><p class="">-- Sean Silva</p></div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left-width:1px;border-left-style:solid;border-left-color:rgb(204,204,204);padding-left:1ex">
<br>
# RUN: rm -rf %t.dir<br>
@@ -10,6 +8,7 @@<br>
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build1/foo.o<br>
# RUN: cd %t.dir<br>
# RUN: ld.lld --hash-style=gnu build1/foo.o -o bar -shared --as-needed --reproduce repro<br>
+# RUN: cpio -id < repro.cpio<br>
# RUN: diff build1/foo.o repro/%:t.dir/build1/foo.o<br>
<br>
# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt<br>
@@ -24,6 +23,7 @@<br>
# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.dir/build2/foo.o<br>
# RUN: cd %t.dir/build2/a/b/c<br>
# RUN: ld.lld ./../../../foo.o -o bar -shared --as-needed --reproduce repro<br>
+# RUN: cpio -id < repro.cpio<br>
# RUN: diff %t.dir/build2/foo.o repro/%:t.dir/build2/foo.o<br>
<br>
# RUN: echo "{ local: *; };" > ver<br>
@@ -33,6 +33,7 @@<br>
# RUN: ld.lld --reproduce repro2 'foo bar' -L"foo bar" -Lfile \<br>
# RUN: --dynamic-list dyn -rpath file --script file --version-script ver \<br>
# RUN: --dynamic-linker "some unusual/path"<br>
+# RUN: cpio -id < repro2.cpio<br>
# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt<br>
# RSP2: "{{.*}}foo bar"<br>
# RSP2-NEXT: -L "{{.*}}foo bar"<br>
@@ -43,10 +44,6 @@<br>
# RSP2-NEXT: --version-script {{.+}}ver<br>
# RSP2-NEXT: --dynamic-linker "some unusual/path"<br>
<br>
-# RUN: not ld.lld build1/foo.o -o bar -shared --as-needed --reproduce . 2>&1 \<br>
-# RUN: | FileCheck --check-prefix=ERROR %s<br>
-# ERROR: can't create directory<br>
-<br>
.globl _start<br>
_start:<br>
mov $60, %rax<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div></div>