r249315 - [VFS] Add an in-memory file system implementation.
Benjamin Kramer via cfe-commits
cfe-commits at lists.llvm.org
Mon Oct 5 06:55:15 PDT 2015
Author: d0k
Date: Mon Oct 5 08:55:14 2015
New Revision: 249315
URL: http://llvm.org/viewvc/llvm-project?rev=249315&view=rev
Log:
[VFS] Add an in-memory file system implementation.
This is a simple file system tree of memory buffers that can be filled by a
client. In conjunction with an OverlayFS it can be used to make virtual
files accessible right next to physical files. This can be used as a
replacement for the virtual file handling in FileManager and which I intend
to remove eventually.
Modified:
cfe/trunk/include/clang/Basic/VirtualFileSystem.h
cfe/trunk/lib/Basic/VirtualFileSystem.cpp
cfe/trunk/unittests/Basic/VirtualFileSystemTest.cpp
cfe/trunk/unittests/Tooling/RewriterTestContext.h
Modified: cfe/trunk/include/clang/Basic/VirtualFileSystem.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/VirtualFileSystem.h?rev=249315&r1=249314&r2=249315&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/VirtualFileSystem.h (original)
+++ cfe/trunk/include/clang/Basic/VirtualFileSystem.h Mon Oct 5 08:55:14 2015
@@ -241,6 +241,27 @@ public:
iterator overlays_end() { return FSList.rend(); }
};
+namespace detail {
+class InMemoryDirectory;
+} // end namespace detail
+
+/// An in-memory file system.
+class InMemoryFileSystem : public FileSystem {
+ std::unique_ptr<detail::InMemoryDirectory> Root;
+
+public:
+ InMemoryFileSystem();
+ ~InMemoryFileSystem() override;
+ void addFile(const Twine &Path, time_t ModificationTime,
+ std::unique_ptr<llvm::MemoryBuffer> Buffer);
+ StringRef toString() const;
+
+ llvm::ErrorOr<Status> status(const Twine &Path) override;
+ llvm::ErrorOr<std::unique_ptr<File>>
+ openFileForRead(const Twine &Path) override;
+ directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
+};
+
/// \brief Get a globally unique ID for a virtual file or directory.
llvm::sys::fs::UniqueID getNextVirtualUniqueID();
Modified: cfe/trunk/lib/Basic/VirtualFileSystem.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/VirtualFileSystem.cpp?rev=249315&r1=249314&r2=249315&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/VirtualFileSystem.cpp (original)
+++ cfe/trunk/lib/Basic/VirtualFileSystem.cpp Mon Oct 5 08:55:14 2015
@@ -320,6 +320,252 @@ directory_iterator OverlayFileSystem::di
std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
}
+namespace clang {
+namespace vfs {
+namespace detail {
+
+enum InMemoryNodeKind { IME_File, IME_Directory };
+
+/// The in memory file system is a tree of Nodes. Every node can either be a
+/// file or a directory.
+class InMemoryNode {
+ Status Stat;
+ InMemoryNodeKind Kind;
+
+public:
+ InMemoryNode(Status Stat, InMemoryNodeKind Kind)
+ : Stat(std::move(Stat)), Kind(Kind) {}
+ virtual ~InMemoryNode() {}
+ const Status &getStatus() const { return Stat; }
+ InMemoryNodeKind getKind() const { return Kind; }
+ virtual std::string toString(unsigned Indent) const = 0;
+};
+
+namespace {
+class InMemoryFile : public InMemoryNode {
+ std::unique_ptr<llvm::MemoryBuffer> Buffer;
+
+public:
+ InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
+ : InMemoryNode(std::move(Stat), IME_File), Buffer(std::move(Buffer)) {}
+
+ llvm::MemoryBuffer *getBuffer() { return Buffer.get(); }
+ std::string toString(unsigned Indent) const override {
+ return (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
+ }
+ static bool classof(const InMemoryNode *N) {
+ return N->getKind() == IME_File;
+ }
+};
+
+/// Adapt a InMemoryFile for VFS' File interface.
+class InMemoryFileAdaptor : public File {
+ InMemoryFile &Node;
+
+public:
+ explicit InMemoryFileAdaptor(InMemoryFile &Node) : Node(Node) {}
+
+ llvm::ErrorOr<Status> status() override { return Node.getStatus(); }
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getBuffer(const Twine &Name, int64_t FileSize = -1,
+ bool RequiresNullTerminator = true,
+ bool IsVolatile = false) override {
+ llvm::MemoryBuffer *Buf = Node.getBuffer();
+ return llvm::MemoryBuffer::getMemBuffer(
+ Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
+ }
+ std::error_code close() override { return std::error_code(); }
+};
+} // end anonymous namespace
+
+class InMemoryDirectory : public InMemoryNode {
+ std::map<std::string, std::unique_ptr<InMemoryNode>> Entries;
+
+public:
+ InMemoryDirectory(Status Stat)
+ : InMemoryNode(std::move(Stat), IME_Directory) {}
+ InMemoryNode *getChild(StringRef Name) {
+ auto I = Entries.find(Name);
+ if (I != Entries.end())
+ return I->second.get();
+ return nullptr;
+ }
+ InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
+ return Entries.insert(make_pair(Name, std::move(Child)))
+ .first->second.get();
+ }
+
+ typedef decltype(Entries)::const_iterator const_iterator;
+ const_iterator begin() const { return Entries.begin(); }
+ const_iterator end() const { return Entries.end(); }
+
+ std::string toString(unsigned Indent) const override {
+ std::string Result =
+ (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
+ for (const auto &Entry : Entries) {
+ Result += Entry.second->toString(Indent + 2);
+ }
+ return Result;
+ }
+ static bool classof(const InMemoryNode *N) {
+ return N->getKind() == IME_Directory;
+ }
+};
+}
+
+InMemoryFileSystem::InMemoryFileSystem()
+ : Root(new detail::InMemoryDirectory(
+ Status("", getNextVirtualUniqueID(), llvm::sys::TimeValue::MinTime(),
+ 0, 0, 0, llvm::sys::fs::file_type::directory_file,
+ llvm::sys::fs::perms::all_all))) {}
+
+InMemoryFileSystem::~InMemoryFileSystem() {}
+
+StringRef InMemoryFileSystem::toString() const {
+ return Root->toString(/*Indent=*/0);
+}
+
+void InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
+ std::unique_ptr<llvm::MemoryBuffer> Buffer) {
+ SmallString<128> Path;
+ P.toVector(Path);
+
+ // Fix up relative paths. This just prepends the current working directory.
+ std::error_code EC = sys::fs::make_absolute(Path);
+ assert(!EC);
+ (void)EC;
+
+ detail::InMemoryDirectory *Dir = Root.get();
+ auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
+ while (true) {
+ StringRef Name = *I;
+ detail::InMemoryNode *Node = Dir->getChild(Name);
+ ++I;
+ if (!Node) {
+ if (I == E) {
+ // End of the path, create a new file.
+ // FIXME: expose the status details in the interface.
+ Status Stat(Path, getNextVirtualUniqueID(),
+ llvm::sys::TimeValue(ModificationTime), 0, 0,
+ Buffer->getBufferSize(),
+ llvm::sys::fs::file_type::regular_file,
+ llvm::sys::fs::all_all);
+ Dir->addChild(Name, llvm::make_unique<detail::InMemoryFile>(
+ std::move(Stat), std::move(Buffer)));
+ return;
+ }
+
+ // Create a new directory. Use the path up to here.
+ // FIXME: expose the status details in the interface.
+ Status Stat(
+ StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
+ getNextVirtualUniqueID(), llvm::sys::TimeValue(ModificationTime), 0,
+ 0, Buffer->getBufferSize(), llvm::sys::fs::file_type::directory_file,
+ llvm::sys::fs::all_all);
+ Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
+ Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
+ continue;
+ }
+
+ if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node))
+ Dir = NewDir;
+ }
+}
+
+static ErrorOr<detail::InMemoryNode *>
+lookupInMemoryNode(detail::InMemoryDirectory *Dir, const Twine &P) {
+ SmallString<128> Path;
+ P.toVector(Path);
+
+ // Fix up relative paths. This just prepends the current working directory.
+ std::error_code EC = sys::fs::make_absolute(Path);
+ assert(!EC);
+ (void)EC;
+
+ auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
+ while (true) {
+ detail::InMemoryNode *Node = Dir->getChild(*I);
+ ++I;
+ if (!Node)
+ return errc::no_such_file_or_directory;
+
+ // Return the file if it's at the end of the path.
+ if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
+ if (I == E)
+ return File;
+ return errc::no_such_file_or_directory;
+ }
+
+ // Traverse directories.
+ Dir = cast<detail::InMemoryDirectory>(Node);
+ if (I == E)
+ return Dir;
+ }
+}
+
+llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
+ auto Node = lookupInMemoryNode(Root.get(), Path);
+ if (Node)
+ return (*Node)->getStatus();
+ return Node.getError();
+}
+
+llvm::ErrorOr<std::unique_ptr<File>>
+InMemoryFileSystem::openFileForRead(const Twine &Path) {
+ auto Node = lookupInMemoryNode(Root.get(), Path);
+ if (!Node)
+ return Node.getError();
+
+ // When we have a file provide a heap-allocated wrapper for the memory buffer
+ // to match the ownership semantics for File.
+ if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
+ return std::unique_ptr<File>(new detail::InMemoryFileAdaptor(*F));
+
+ // FIXME: errc::not_a_file?
+ return make_error_code(llvm::errc::invalid_argument);
+}
+
+namespace {
+/// Adaptor from InMemoryDir::iterator to directory_iterator.
+class InMemoryDirIterator : public clang::vfs::detail::DirIterImpl {
+ detail::InMemoryDirectory::const_iterator I;
+ detail::InMemoryDirectory::const_iterator E;
+
+public:
+ InMemoryDirIterator() {}
+ explicit InMemoryDirIterator(detail::InMemoryDirectory &Dir)
+ : I(Dir.begin()), E(Dir.end()) {
+ if (I != E)
+ CurrentEntry = I->second->getStatus();
+ }
+
+ std::error_code increment() override {
+ ++I;
+ // When we're at the end, make CurrentEntry invalid and DirIterImpl will do
+ // the rest.
+ CurrentEntry = I != E ? I->second->getStatus() : Status();
+ return std::error_code();
+ }
+};
+} // end anonymous namespace
+
+directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
+ std::error_code &EC) {
+ auto Node = lookupInMemoryNode(Root.get(), Dir);
+ if (!Node) {
+ EC = Node.getError();
+ return directory_iterator(std::make_shared<InMemoryDirIterator>());
+ }
+
+ if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
+ return directory_iterator(std::make_shared<InMemoryDirIterator>(*DirNode));
+
+ EC = make_error_code(llvm::errc::not_a_directory);
+ return directory_iterator(std::make_shared<InMemoryDirIterator>());
+}
+}
+}
+
//===-----------------------------------------------------------------------===/
// VFSFromYAML implementation
//===-----------------------------------------------------------------------===/
Modified: cfe/trunk/unittests/Basic/VirtualFileSystemTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Basic/VirtualFileSystemTest.cpp?rev=249315&r1=249314&r2=249315&view=diff
==============================================================================
--- cfe/trunk/unittests/Basic/VirtualFileSystemTest.cpp (original)
+++ cfe/trunk/unittests/Basic/VirtualFileSystemTest.cpp Mon Oct 5 08:55:14 2015
@@ -515,6 +515,73 @@ TEST(VirtualFileSystemTest, HiddenInIter
}
}
+class InMemoryFileSystemTest : public ::testing::Test {
+protected:
+ clang::vfs::InMemoryFileSystem FS;
+};
+
+TEST_F(InMemoryFileSystemTest, IsEmpty) {
+ auto Stat = FS.status("/a");
+ ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString();
+ Stat = FS.status("/");
+ ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, WindowsPath) {
+ FS.addFile("c:/windows/system128/foo.cpp", 0, MemoryBuffer::getMemBuffer(""));
+ auto Stat = FS.status("c:");
+ ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+ Stat = FS.status("c:/windows/system128/foo.cpp");
+ ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+ FS.addFile("d:/windows/foo.cpp", 0, MemoryBuffer::getMemBuffer(""));
+ Stat = FS.status("d:/windows/foo.cpp");
+ ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, OverlayFile) {
+ FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a"));
+ auto Stat = FS.status("/");
+ ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+ Stat = FS.status("/a");
+ ASSERT_FALSE(Stat.getError()) << Stat.getError() << "\n" << FS.toString();
+ ASSERT_EQ("/a", Stat->getName());
+}
+
+TEST_F(InMemoryFileSystemTest, OpenFileForRead) {
+ FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a"));
+ auto File = FS.openFileForRead("/a");
+ ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer());
+ File = FS.openFileForRead("/a"); // Open again.
+ ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer());
+ File = FS.openFileForRead("/");
+ ASSERT_EQ(errc::invalid_argument, File.getError()) << FS.toString();
+ File = FS.openFileForRead("/b");
+ ASSERT_EQ(errc::no_such_file_or_directory, File.getError()) << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, DirectoryIteration) {
+ FS.addFile("/a", 0, MemoryBuffer::getMemBuffer(""));
+ FS.addFile("/b/c", 0, MemoryBuffer::getMemBuffer(""));
+
+ std::error_code EC;
+ vfs::directory_iterator I = FS.dir_begin("/", EC);
+ ASSERT_FALSE(EC);
+ ASSERT_EQ("/a", I->getName());
+ I.increment(EC);
+ ASSERT_FALSE(EC);
+ ASSERT_EQ("/b", I->getName());
+ I.increment(EC);
+ ASSERT_FALSE(EC);
+ ASSERT_EQ(vfs::directory_iterator(), I);
+
+ I = FS.dir_begin("/b", EC);
+ ASSERT_FALSE(EC);
+ ASSERT_EQ("/b/c", I->getName());
+ I.increment(EC);
+ ASSERT_FALSE(EC);
+ ASSERT_EQ(vfs::directory_iterator(), I);
+}
+
// NOTE: in the tests below, we use '//root/' as our root directory, since it is
// a legal *absolute* path on Windows as well as *nix.
class VFSFromYAMLTest : public ::testing::Test {
Modified: cfe/trunk/unittests/Tooling/RewriterTestContext.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/RewriterTestContext.h?rev=249315&r1=249314&r2=249315&view=diff
==============================================================================
--- cfe/trunk/unittests/Tooling/RewriterTestContext.h (original)
+++ cfe/trunk/unittests/Tooling/RewriterTestContext.h Mon Oct 5 08:55:14 2015
@@ -34,15 +34,20 @@ namespace clang {
/// methods, which help with writing tests that change files.
class RewriterTestContext {
public:
- RewriterTestContext()
- : DiagOpts(new DiagnosticOptions()),
- Diagnostics(IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
- &*DiagOpts),
- DiagnosticPrinter(llvm::outs(), &*DiagOpts),
- Files((FileSystemOptions())),
- Sources(Diagnostics, Files),
- Rewrite(Sources, Options) {
+ RewriterTestContext()
+ : DiagOpts(new DiagnosticOptions()),
+ Diagnostics(IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
+ &*DiagOpts),
+ DiagnosticPrinter(llvm::outs(), &*DiagOpts),
+ InMemoryFileSystem(new vfs::InMemoryFileSystem),
+ OverlayFileSystem(
+ new vfs::OverlayFileSystem(vfs::getRealFileSystem())),
+ Files(FileSystemOptions(), OverlayFileSystem),
+ Sources(Diagnostics, Files), Rewrite(Sources, Options) {
Diagnostics.setClient(&DiagnosticPrinter, false);
+ // FIXME: To make these tests truly in-memory, we need to overlay the
+ // builtin headers.
+ OverlayFileSystem->pushOverlay(InMemoryFileSystem);
}
~RewriterTestContext() {}
@@ -50,9 +55,9 @@ class RewriterTestContext {
FileID createInMemoryFile(StringRef Name, StringRef Content) {
std::unique_ptr<llvm::MemoryBuffer> Source =
llvm::MemoryBuffer::getMemBuffer(Content);
- const FileEntry *Entry =
- Files.getVirtualFile(Name, Source->getBufferSize(), 0);
- Sources.overrideFileContents(Entry, std::move(Source));
+ InMemoryFileSystem->addFile(Name, 0, std::move(Source));
+
+ const FileEntry *Entry = Files.getFile(Name);
assert(Entry != nullptr);
return Sources.createFileID(Entry, SourceLocation(), SrcMgr::C_User);
}
@@ -109,6 +114,8 @@ class RewriterTestContext {
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
DiagnosticsEngine Diagnostics;
TextDiagnosticPrinter DiagnosticPrinter;
+ IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem;
+ IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem;
FileManager Files;
SourceManager Sources;
LangOptions Options;
More information about the cfe-commits
mailing list