[llvm] r366956 - [Support] move FileCollector from LLDB to llvm/Support
Alex Lorenz via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 24 15:59:20 PDT 2019
Author: arphaman
Date: Wed Jul 24 15:59:20 2019
New Revision: 366956
URL: http://llvm.org/viewvc/llvm-project?rev=366956&view=rev
Log:
[Support] move FileCollector from LLDB to llvm/Support
The file collector class is useful for creating reproducers,
not just for LLDB, but for other tools as well in LLVM/Clang.
Differential Revision: https://reviews.llvm.org/D65237
Added:
llvm/trunk/include/llvm/Support/FileCollector.h
llvm/trunk/lib/Support/FileCollector.cpp
llvm/trunk/unittests/Support/FileCollectorTest.cpp
Modified:
llvm/trunk/lib/Support/CMakeLists.txt
llvm/trunk/unittests/Support/CMakeLists.txt
Added: llvm/trunk/include/llvm/Support/FileCollector.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/FileCollector.h?rev=366956&view=auto
==============================================================================
--- llvm/trunk/include/llvm/Support/FileCollector.h (added)
+++ llvm/trunk/include/llvm/Support/FileCollector.h Wed Jul 24 15:59:20 2019
@@ -0,0 +1,74 @@
+//===-- FileCollector.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_FILE_COLLECTOR_H
+#define LLVM_SUPPORT_FILE_COLLECTOR_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+#include <mutex>
+
+namespace llvm {
+
+/// Collects files into a directory and generates a mapping that can be used by
+/// the VFS.
+class FileCollector {
+public:
+ FileCollector(std::string root, std::string overlay);
+
+ void AddFile(const Twine &file);
+
+ /// Write the yaml mapping (for the VFS) to the given file.
+ std::error_code WriteMapping(StringRef mapping_file);
+
+ /// Copy the files into the root directory.
+ ///
+ /// When stop_on_error is true (the default) we abort as soon as one file
+ /// cannot be copied. This is relatively common, for example when a file was
+ /// removed after it was added to the mapping.
+ std::error_code CopyFiles(bool stop_on_error = true);
+
+private:
+ void AddFileImpl(StringRef src_path);
+
+ bool MarkAsSeen(StringRef path) { return m_seen.insert(path).second; }
+
+ bool GetRealPath(StringRef src_path,
+ SmallVectorImpl<char> &result);
+
+ void AddFileToMapping(StringRef virtual_path,
+ StringRef real_path) {
+ m_vfs_writer.addFileMapping(virtual_path, real_path);
+ }
+
+ /// Synchronizes adding files.
+ std::mutex m_mutex;
+
+ /// The root directory where files are copied.
+ std::string m_root;
+
+ /// The root directory where the VFS overlay lives.
+ std::string m_overlay_root;
+
+ /// Tracks already seen files so they can be skipped.
+ StringSet<> m_seen;
+
+ /// The yaml mapping writer.
+ vfs::YAMLVFSWriter m_vfs_writer;
+
+ /// Caches real_path calls when resolving symlinks.
+ StringMap<std::string> m_symlink_map;
+};
+
+} // end namespace llvm
+
+#endif // LLVM_SUPPORT_FILE_COLLECTOR_H
Modified: llvm/trunk/lib/Support/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CMakeLists.txt?rev=366956&r1=366955&r2=366956&view=diff
==============================================================================
--- llvm/trunk/lib/Support/CMakeLists.txt (original)
+++ llvm/trunk/lib/Support/CMakeLists.txt Wed Jul 24 15:59:20 2019
@@ -89,6 +89,7 @@ add_llvm_library(LLVMSupport
Error.cpp
ErrorHandling.cpp
FileCheck.cpp
+ FileCollector.cpp
FileUtilities.cpp
FileOutputBuffer.cpp
FoldingSet.cpp
Added: llvm/trunk/lib/Support/FileCollector.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/FileCollector.cpp?rev=366956&view=auto
==============================================================================
--- llvm/trunk/lib/Support/FileCollector.cpp (added)
+++ llvm/trunk/lib/Support/FileCollector.cpp Wed Jul 24 15:59:20 2019
@@ -0,0 +1,177 @@
+//===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/FileCollector.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+
+using namespace llvm;
+
+static bool IsCaseSensitivePath(StringRef path) {
+ SmallString<256> tmp_dest = path, upper_dest, real_dest;
+
+ // Remove component traversals, links, etc.
+ if (!sys::fs::real_path(path, tmp_dest))
+ return true; // Current default value in vfs.yaml
+ path = tmp_dest;
+
+ // Change path to all upper case and ask for its real path, if the latter
+ // exists and is equal to path, it's not case sensitive. Default to case
+ // sensitive in the absence of real_path, since this is the YAMLVFSWriter
+ // default.
+ upper_dest = path.upper();
+ if (sys::fs::real_path(upper_dest, real_dest) && path.equals(real_dest))
+ return false;
+ return true;
+}
+
+FileCollector::FileCollector(std::string root, std::string overlay_root)
+ : m_root(std::move(root)), m_overlay_root(std::move(overlay_root)) {
+ sys::fs::create_directories(this->m_root, true);
+}
+
+bool FileCollector::GetRealPath(StringRef src_path,
+ SmallVectorImpl<char> &result) {
+ SmallString<256> real_path;
+ StringRef FileName = sys::path::filename(src_path);
+ std::string directory = sys::path::parent_path(src_path).str();
+ auto dir_with_symlink = m_symlink_map.find(directory);
+
+ // Use real_path to fix any symbolic link component present in a path.
+ // Computing the real path is expensive, cache the search through the
+ // parent path directory.
+ if (dir_with_symlink == m_symlink_map.end()) {
+ auto ec = sys::fs::real_path(directory, real_path);
+ if (ec)
+ return false;
+ m_symlink_map[directory] = real_path.str();
+ } else {
+ real_path = dir_with_symlink->second;
+ }
+
+ sys::path::append(real_path, FileName);
+ result.swap(real_path);
+ return true;
+}
+
+void FileCollector::AddFile(const Twine &file) {
+ std::lock_guard<std::mutex> lock(m_mutex);
+ std::string file_str = file.str();
+ if (MarkAsSeen(file_str))
+ AddFileImpl(file_str);
+}
+
+void FileCollector::AddFileImpl(StringRef src_path) {
+ // We need an absolute src path to append to the root.
+ SmallString<256> absolute_src = src_path;
+ sys::fs::make_absolute(absolute_src);
+
+ // Canonicalize src to a native path to avoid mixed separator styles.
+ sys::path::native(absolute_src);
+
+ // Remove redundant leading "./" pieces and consecutive separators.
+ absolute_src = sys::path::remove_leading_dotslash(absolute_src);
+
+ // Canonicalize the source path by removing "..", "." components.
+ SmallString<256> virtual_path = absolute_src;
+ sys::path::remove_dots(virtual_path, /*remove_dot_dot=*/true);
+
+ // If a ".." component is present after a symlink component, remove_dots may
+ // lead to the wrong real destination path. Let the source be canonicalized
+ // like that but make sure we always use the real path for the destination.
+ SmallString<256> copy_from;
+ if (!GetRealPath(absolute_src, copy_from))
+ copy_from = virtual_path;
+
+ SmallString<256> dst_path = StringRef(m_root);
+ sys::path::append(dst_path, sys::path::relative_path(copy_from));
+
+ // Always map a canonical src path to its real path into the YAML, by doing
+ // this we map different virtual src paths to the same entry in the VFS
+ // overlay, which is a way to emulate symlink inside the VFS; this is also
+ // needed for correctness, not doing that can lead to module redefinition
+ // errors.
+ AddFileToMapping(virtual_path, dst_path);
+}
+
+/// Set the access and modification time for the given file from the given
+/// status object.
+static std::error_code
+CopyAccessAndModificationTime(StringRef filename,
+ const sys::fs::file_status &stat) {
+ int fd;
+
+ if (auto ec =
+ sys::fs::openFileForWrite(filename, fd, sys::fs::CD_OpenExisting))
+ return ec;
+
+ if (auto ec = sys::fs::setLastAccessAndModificationTime(
+ fd, stat.getLastAccessedTime(), stat.getLastModificationTime()))
+ return ec;
+
+ if (auto ec = sys::Process::SafelyCloseFileDescriptor(fd))
+ return ec;
+
+ return {};
+}
+
+std::error_code FileCollector::CopyFiles(bool stop_on_error) {
+ for (auto &entry : m_vfs_writer.getMappings()) {
+ // Create directory tree.
+ if (std::error_code ec =
+ sys::fs::create_directories(sys::path::parent_path(entry.RPath),
+ /*IgnoreExisting=*/true)) {
+ if (stop_on_error)
+ return ec;
+ }
+
+ // Copy file over.
+ if (std::error_code ec = sys::fs::copy_file(entry.VPath, entry.RPath)) {
+ if (stop_on_error)
+ return ec;
+ }
+
+ // Copy over permissions.
+ if (auto perms = sys::fs::getPermissions(entry.VPath)) {
+ if (std::error_code ec = sys::fs::setPermissions(entry.RPath, *perms)) {
+ if (stop_on_error)
+ return ec;
+ }
+ }
+
+ // Copy over modification time.
+ sys::fs::file_status stat;
+ if (std::error_code ec = sys::fs::status(entry.VPath, stat)) {
+ if (stop_on_error)
+ return ec;
+ continue;
+ }
+ CopyAccessAndModificationTime(entry.RPath, stat);
+ }
+ return {};
+}
+
+std::error_code FileCollector::WriteMapping(StringRef mapping_file) {
+ std::lock_guard<std::mutex> lock(m_mutex);
+
+ StringRef root = m_overlay_root;
+ m_vfs_writer.setOverlayDir(root);
+ m_vfs_writer.setCaseSensitivity(IsCaseSensitivePath(root));
+ m_vfs_writer.setUseExternalNames(false);
+
+ std::error_code ec;
+ raw_fd_ostream os(mapping_file, ec, sys::fs::F_Text);
+ if (ec)
+ return ec;
+
+ m_vfs_writer.write(os);
+
+ return {};
+}
Modified: llvm/trunk/unittests/Support/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Support/CMakeLists.txt?rev=366956&r1=366955&r2=366956&view=diff
==============================================================================
--- llvm/trunk/unittests/Support/CMakeLists.txt (original)
+++ llvm/trunk/unittests/Support/CMakeLists.txt Wed Jul 24 15:59:20 2019
@@ -30,6 +30,7 @@ add_llvm_unittest(SupportTests
ErrorOrTest.cpp
ErrorTest.cpp
FileCheckTest.cpp
+ FileCollectorTest.cpp
FileOutputBufferTest.cpp
FormatVariadicTest.cpp
GlobPatternTest.cpp
Added: llvm/trunk/unittests/Support/FileCollectorTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Support/FileCollectorTest.cpp?rev=366956&view=auto
==============================================================================
--- llvm/trunk/unittests/Support/FileCollectorTest.cpp (added)
+++ llvm/trunk/unittests/Support/FileCollectorTest.cpp Wed Jul 24 15:59:20 2019
@@ -0,0 +1,211 @@
+//===-- FileCollectorTest.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "llvm/Support/FileCollector.h"
+#include "llvm/Support/FileSystem.h"
+
+using namespace llvm;
+
+namespace llvm {
+namespace vfs {
+inline bool operator==(const llvm::vfs::YAMLVFSEntry &LHS,
+ const llvm::vfs::YAMLVFSEntry &RHS) {
+ return LHS.VPath == RHS.VPath && LHS.RPath == RHS.RPath;
+}
+} // namespace vfs
+} // namespace llvm
+
+namespace {
+class TestingFileCollector : public FileCollector {
+public:
+ using FileCollector::FileCollector;
+ using FileCollector::m_root;
+ using FileCollector::m_seen;
+ using FileCollector::m_symlink_map;
+ using FileCollector::m_vfs_writer;
+
+ bool HasSeen(StringRef fs) {
+ return m_seen.find(fs) != m_seen.end();
+ }
+};
+
+struct ScopedDir {
+ SmallString<128> Path;
+ ScopedDir(const Twine &Name, bool Unique = false) {
+ std::error_code EC;
+ if (Unique) {
+ EC = llvm::sys::fs::createUniqueDirectory(Name, Path);
+ } else {
+ Path = Name.str();
+ EC = llvm::sys::fs::create_directory(Twine(Path));
+ }
+ if (EC)
+ Path = "";
+ EXPECT_FALSE(EC);
+ // Ensure the path is the real path so tests can use it to compare against
+ // realpath output.
+ SmallString<128> RealPath;
+ if (!llvm::sys::fs::real_path(Path, RealPath))
+ Path.swap(RealPath);
+ }
+ ~ScopedDir() {
+ if (Path != "") {
+ EXPECT_FALSE(llvm::sys::fs::remove_directories(Path.str()));
+ }
+ }
+ operator StringRef() { return Path.str(); }
+};
+
+struct ScopedLink {
+ SmallString<128> Path;
+ ScopedLink(const Twine &To, const Twine &From) {
+ Path = From.str();
+ std::error_code EC = sys::fs::create_link(To, From);
+ if (EC)
+ Path = "";
+ EXPECT_FALSE(EC);
+ }
+ ~ScopedLink() {
+ if (Path != "") {
+ EXPECT_FALSE(llvm::sys::fs::remove(Path.str()));
+ }
+ }
+ operator StringRef() { return Path.str(); }
+};
+
+struct ScopedFile {
+ SmallString<128> Path;
+ ScopedFile(const Twine &Name) {
+ std::error_code EC;
+ EC = llvm::sys::fs::createUniqueFile(Name, Path);
+ if (EC)
+ Path = "";
+ EXPECT_FALSE(EC);
+ }
+ ~ScopedFile() {
+ if (Path != "") {
+ EXPECT_FALSE(llvm::sys::fs::remove(Path.str()));
+ }
+ }
+ operator StringRef() { return Path.str(); }
+};
+} // end anonymous namespace
+
+TEST(FileCollectorTest, AddFile) {
+ ScopedDir root("add_file_root", true);
+ std::string root_fs = root.Path.str();
+ TestingFileCollector file_collector(root_fs, root_fs);
+
+ file_collector.AddFile("/path/to/a");
+ file_collector.AddFile("/path/to/b");
+ file_collector.AddFile("/path/to/c");
+
+ // Make sure the root is correct.
+ EXPECT_EQ(file_collector.m_root, root_fs);
+
+ // Make sure we've seen all the added files.
+ EXPECT_TRUE(file_collector.HasSeen("/path/to/a"));
+ EXPECT_TRUE(file_collector.HasSeen("/path/to/b"));
+ EXPECT_TRUE(file_collector.HasSeen("/path/to/c"));
+
+ // Make sure we've only seen the added files.
+ EXPECT_FALSE(file_collector.HasSeen("/path/to/d"));
+}
+
+TEST(FileCollectorTest, CopyFiles) {
+ ScopedDir file_root("file_root", true);
+ ScopedFile a(file_root + "/aaa");
+ ScopedFile b(file_root + "/bbb");
+ ScopedFile c(file_root + "/ccc");
+
+ // Create file collector and add files.
+ ScopedDir root("copy_files_root", true);
+ std::string root_fs = root.Path.str();
+ TestingFileCollector file_collector(root_fs, root_fs);
+ file_collector.AddFile(a.Path);
+ file_collector.AddFile(b.Path);
+ file_collector.AddFile(c.Path);
+
+ // Make sure we can copy the files.
+ std::error_code ec = file_collector.CopyFiles(true);
+ EXPECT_FALSE(ec);
+
+ // Now add a bogus file and make sure we error out.
+ file_collector.AddFile("/some/bogus/file");
+ ec = file_collector.CopyFiles(true);
+ EXPECT_TRUE(ec);
+
+ // However, if stop_on_error is true the copy should still succeed.
+ ec = file_collector.CopyFiles(false);
+ EXPECT_FALSE(ec);
+}
+
+#ifndef _WIN32
+TEST(FileCollectorTest, Symlinks) {
+ // Root where the original files live.
+ ScopedDir file_root("file_root", true);
+
+ // Create some files in the file root.
+ ScopedFile a(file_root + "/aaa");
+ ScopedFile b(file_root + "/bbb");
+ ScopedFile c(file_root + "/ccc");
+
+ // Create a directory foo with file ddd.
+ ScopedDir foo(file_root + "/foo");
+ ScopedFile d(foo + "/ddd");
+
+ // Create a file eee in the foo's parent directory.
+ ScopedFile e(foo + "/../eee");
+
+ // Create a symlink bar pointing to foo.
+ ScopedLink symlink(file_root + "/foo", file_root + "/bar");
+
+ // Root where files are copied to.
+ ScopedDir reproducer_root("reproducer_root", true);
+ std::string root_fs = reproducer_root.Path.str();
+ TestingFileCollector file_collector(root_fs, root_fs);
+
+ // Add all the files to the collector.
+ file_collector.AddFile(a.Path);
+ file_collector.AddFile(b.Path);
+ file_collector.AddFile(c.Path);
+ file_collector.AddFile(d.Path);
+ file_collector.AddFile(e.Path);
+ file_collector.AddFile(file_root + "/bar/ddd");
+
+ auto mapping = file_collector.m_vfs_writer.getMappings();
+
+ {
+ // Make sure the common case works.
+ std::string vpath = (file_root + "/aaa").str();
+ std::string rpath = (reproducer_root.Path + file_root.Path + "/aaa").str();
+ printf("%s -> %s\n", vpath.c_str(), rpath.c_str());
+ EXPECT_THAT(mapping, testing::Contains(vfs::YAMLVFSEntry(vpath, rpath)));
+ }
+
+ {
+ // Make sure the virtual path points to the real source path.
+ std::string vpath = (file_root + "/bar/ddd").str();
+ std::string rpath =
+ (reproducer_root.Path + file_root.Path + "/foo/ddd").str();
+ printf("%s -> %s\n", vpath.c_str(), rpath.c_str());
+ EXPECT_THAT(mapping, testing::Contains(vfs::YAMLVFSEntry(vpath, rpath)));
+ }
+
+ {
+ // Make sure that .. is removed from the source path.
+ std::string vpath = (file_root + "/eee").str();
+ std::string rpath = (reproducer_root.Path + file_root.Path + "/eee").str();
+ printf("%s -> %s\n", vpath.c_str(), rpath.c_str());
+ EXPECT_THAT(mapping, testing::Contains(vfs::YAMLVFSEntry(vpath, rpath)));
+ }
+}
+#endif
More information about the llvm-commits
mailing list