[llvm] r345431 - [VFS] Add property 'fallthrough' that controls fallback to real file system.
Volodymyr Sapsai via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 26 15:14:34 PDT 2018
Author: vsapsai
Date: Fri Oct 26 15:14:33 2018
New Revision: 345431
URL: http://llvm.org/viewvc/llvm-project?rev=345431&view=rev
Log:
[VFS] Add property 'fallthrough' that controls fallback to real file system.
Default property value 'true' preserves current behavior. Value 'false' can be
used to create VFS "root", file system that gives better control over which
files compiler can use during compilation as there are no unpredictable
accesses to real file system.
Non-fallthrough use case changes how we treat multiple VFS overlay
files. Instead of all of them being at the same level just above a real
file system, now they are nested and subsequent overlays can refer to
files in previous overlays.
rdar://problem/39465552
Reviewers: bruno, benlangmuir
Reviewed By: bruno
Subscribers: dexonsmith, cfe-commits, hiraditya
Differential Revision: https://reviews.llvm.org/D50539
Modified:
llvm/trunk/lib/Support/VirtualFileSystem.cpp
llvm/trunk/unittests/Support/VirtualFileSystemTest.cpp
Modified: llvm/trunk/lib/Support/VirtualFileSystem.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/VirtualFileSystem.cpp?rev=345431&r1=345430&r2=345431&view=diff
==============================================================================
--- llvm/trunk/lib/Support/VirtualFileSystem.cpp (original)
+++ llvm/trunk/lib/Support/VirtualFileSystem.cpp Fri Oct 26 15:14:33 2018
@@ -993,16 +993,44 @@ public:
static bool classof(const Entry *E) { return E->getKind() == EK_File; }
};
+// FIXME: reuse implementation common with OverlayFSDirIterImpl as these
+// iterators are conceptually similar.
class VFSFromYamlDirIterImpl : public llvm::vfs::detail::DirIterImpl {
std::string Dir;
RedirectingDirectoryEntry::iterator Current, End;
- std::error_code incrementImpl();
+ // To handle 'fallthrough' mode we need to iterate at first through
+ // RedirectingDirectoryEntry and then through ExternalFS. These operations are
+ // done sequentially, we just need to keep a track of what kind of iteration
+ // we are currently performing.
+
+ /// Flag telling if we should iterate through ExternalFS or stop at the last
+ /// RedirectingDirectoryEntry::iterator.
+ bool IterateExternalFS;
+ /// Flag telling if we have switched to iterating through ExternalFS.
+ bool IsExternalFSCurrent = false;
+ FileSystem &ExternalFS;
+ directory_iterator ExternalDirIter;
+ llvm::StringSet<> SeenNames;
+
+ /// To combine multiple iterations, different methods are responsible for
+ /// different iteration steps.
+ /// @{
+
+ /// Responsible for dispatching between RedirectingDirectoryEntry iteration
+ /// and ExternalFS iteration.
+ std::error_code incrementImpl(bool IsFirstTime);
+ /// Responsible for RedirectingDirectoryEntry iteration.
+ std::error_code incrementContent(bool IsFirstTime);
+ /// Responsible for ExternalFS iteration.
+ std::error_code incrementExternal();
+ /// @}
public:
VFSFromYamlDirIterImpl(const Twine &Path,
RedirectingDirectoryEntry::iterator Begin,
RedirectingDirectoryEntry::iterator End,
+ bool IterateExternalFS, FileSystem &ExternalFS,
std::error_code &EC);
std::error_code increment() override;
@@ -1028,6 +1056,7 @@ public:
/// 'case-sensitive': <boolean, default=true>
/// 'use-external-names': <boolean, default=true>
/// 'overlay-relative': <boolean, default=false>
+/// 'fallthrough': <boolean, default=true>
///
/// Virtual directories are represented as
/// \verbatim
@@ -1091,6 +1120,10 @@ class RedirectingFileSystem : public vfs
/// Whether to use to use the value of 'external-contents' for the
/// names of files. This global value is overridable on a per-file basis.
bool UseExternalNames = true;
+
+ /// Whether to attempt a file lookup in external file system after it wasn't
+ /// found in VFS.
+ bool IsFallthrough = true;
/// @}
/// Virtual file paths and external files could be canonicalized without "..",
@@ -1141,6 +1174,8 @@ public:
ErrorOr<Entry *> E = lookupPath(Dir);
if (!E) {
EC = E.getError();
+ if (IsFallthrough && EC == errc::no_such_file_or_directory)
+ return ExternalFS->dir_begin(Dir, EC);
return {};
}
ErrorOr<Status> S = status(Dir, *E);
@@ -1156,7 +1191,8 @@ public:
auto *D = cast<RedirectingDirectoryEntry>(*E);
return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(
- Dir, D->contents_begin(), D->contents_end(), EC));
+ Dir, D->contents_begin(), D->contents_end(),
+ /*IterateExternalFS=*/IsFallthrough, *ExternalFS, EC));
}
void setExternalContentsPrefixDir(StringRef PrefixDir) {
@@ -1538,6 +1574,7 @@ public:
KeyStatusPair("case-sensitive", false),
KeyStatusPair("use-external-names", false),
KeyStatusPair("overlay-relative", false),
+ KeyStatusPair("fallthrough", false),
KeyStatusPair("roots", true),
};
@@ -1595,6 +1632,9 @@ public:
} else if (Key == "use-external-names") {
if (!parseScalarBool(I.getValue(), FS->UseExternalNames))
return false;
+ } else if (Key == "fallthrough") {
+ if (!parseScalarBool(I.getValue(), FS->IsFallthrough))
+ return false;
} else {
llvm_unreachable("key missing from Keys");
}
@@ -1760,8 +1800,13 @@ ErrorOr<Status> RedirectingFileSystem::s
ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
ErrorOr<Entry *> Result = lookupPath(Path);
- if (!Result)
+ if (!Result) {
+ if (IsFallthrough &&
+ Result.getError() == llvm::errc::no_such_file_or_directory) {
+ return ExternalFS->status(Path);
+ }
return Result.getError();
+ }
return status(Path, *Result);
}
@@ -1793,8 +1838,13 @@ public:
ErrorOr<std::unique_ptr<File>>
RedirectingFileSystem::openFileForRead(const Twine &Path) {
ErrorOr<Entry *> E = lookupPath(Path);
- if (!E)
+ if (!E) {
+ if (IsFallthrough &&
+ E.getError() == llvm::errc::no_such_file_or_directory) {
+ return ExternalFS->openFileForRead(Path);
+ }
return E.getError();
+ }
auto *F = dyn_cast<RedirectingFileEntry>(*E);
if (!F) // FIXME: errc::not_a_file?
@@ -2035,18 +2085,42 @@ void YAMLVFSWriter::write(llvm::raw_ostr
VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
const Twine &_Path, RedirectingDirectoryEntry::iterator Begin,
- RedirectingDirectoryEntry::iterator End, std::error_code &EC)
- : Dir(_Path.str()), Current(Begin), End(End) {
- EC = incrementImpl();
+ RedirectingDirectoryEntry::iterator End, bool IterateExternalFS,
+ FileSystem &ExternalFS, std::error_code &EC)
+ : Dir(_Path.str()), Current(Begin), End(End),
+ IterateExternalFS(IterateExternalFS), ExternalFS(ExternalFS) {
+ EC = incrementImpl(/*IsFirstTime=*/true);
}
std::error_code VFSFromYamlDirIterImpl::increment() {
- assert(Current != End && "cannot iterate past end");
- ++Current;
- return incrementImpl();
+ return incrementImpl(/*IsFirstTime=*/false);
}
-std::error_code VFSFromYamlDirIterImpl::incrementImpl() {
+std::error_code VFSFromYamlDirIterImpl::incrementExternal() {
+ assert(!(IsExternalFSCurrent && ExternalDirIter == directory_iterator()) &&
+ "incrementing past end");
+ std::error_code EC;
+ if (IsExternalFSCurrent) {
+ ExternalDirIter.increment(EC);
+ } else if (IterateExternalFS) {
+ ExternalDirIter = ExternalFS.dir_begin(Dir, EC);
+ IsExternalFSCurrent = true;
+ if (EC && EC != errc::no_such_file_or_directory)
+ return EC;
+ EC = {};
+ }
+ if (EC || ExternalDirIter == directory_iterator()) {
+ CurrentEntry = directory_entry();
+ } else {
+ CurrentEntry = *ExternalDirIter;
+ }
+ return EC;
+}
+
+std::error_code VFSFromYamlDirIterImpl::incrementContent(bool IsFirstTime) {
+ assert(IsFirstTime || Current != End && "cannot iterate past end");
+ if (!IsFirstTime)
+ ++Current;
while (Current != End) {
SmallString<128> PathStr(Dir);
llvm::sys::path::append(PathStr, (*Current)->getName());
@@ -2060,12 +2134,22 @@ std::error_code VFSFromYamlDirIterImpl::
break;
}
CurrentEntry = directory_entry(PathStr.str(), Type);
- break;
+ return {};
}
+ return incrementExternal();
+}
- if (Current == End)
- CurrentEntry = directory_entry();
- return {};
+std::error_code VFSFromYamlDirIterImpl::incrementImpl(bool IsFirstTime) {
+ while (true) {
+ std::error_code EC = IsExternalFSCurrent ? incrementExternal()
+ : incrementContent(IsFirstTime);
+ if (EC || CurrentEntry.path().empty())
+ return EC;
+ StringRef Name = llvm::sys::path::filename(CurrentEntry.path());
+ if (SeenNames.insert(Name).second)
+ return EC; // name not seen before
+ }
+ llvm_unreachable("returned above");
}
vfs::recursive_directory_iterator::recursive_directory_iterator(
Modified: llvm/trunk/unittests/Support/VirtualFileSystemTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Support/VirtualFileSystemTest.cpp?rev=345431&r1=345430&r2=345431&view=diff
==============================================================================
--- llvm/trunk/unittests/Support/VirtualFileSystemTest.cpp (original)
+++ llvm/trunk/unittests/Support/VirtualFileSystemTest.cpp Fri Oct 26 15:14:33 2018
@@ -1599,3 +1599,89 @@ TEST_F(VFSFromYAMLTest, RelativePaths) {
EXPECT_EQ(3, NumDiagnostics);
}
+
+TEST_F(VFSFromYAMLTest, NonFallthroughDirectoryIteration) {
+ IntrusiveRefCntPtr<DummyFileSystem> Lower(new DummyFileSystem());
+ Lower->addDirectory("//root/");
+ Lower->addRegularFile("//root/a");
+ Lower->addRegularFile("//root/b");
+ IntrusiveRefCntPtr<vfs::FileSystem> FS = getFromYAMLString(
+ "{ 'use-external-names': false,\n"
+ " 'fallthrough': false,\n"
+ " 'roots': [\n"
+ "{\n"
+ " 'type': 'directory',\n"
+ " 'name': '//root/',\n"
+ " 'contents': [ {\n"
+ " 'type': 'file',\n"
+ " 'name': 'c',\n"
+ " 'external-contents': '//root/a'\n"
+ " }\n"
+ " ]\n"
+ "}\n"
+ "]\n"
+ "}",
+ Lower);
+ ASSERT_TRUE(FS.get() != nullptr);
+
+ std::error_code EC;
+ checkContents(FS->dir_begin("//root/", EC),
+ {"//root/c"});
+}
+
+TEST_F(VFSFromYAMLTest, DirectoryIterationWithDuplicates) {
+ IntrusiveRefCntPtr<DummyFileSystem> Lower(new DummyFileSystem());
+ Lower->addDirectory("//root/");
+ Lower->addRegularFile("//root/a");
+ Lower->addRegularFile("//root/b");
+ IntrusiveRefCntPtr<vfs::FileSystem> FS = getFromYAMLString(
+ "{ 'use-external-names': false,\n"
+ " 'roots': [\n"
+ "{\n"
+ " 'type': 'directory',\n"
+ " 'name': '//root/',\n"
+ " 'contents': [ {\n"
+ " 'type': 'file',\n"
+ " 'name': 'a',\n"
+ " 'external-contents': '//root/a'\n"
+ " }\n"
+ " ]\n"
+ "}\n"
+ "]\n"
+ "}",
+ Lower);
+ ASSERT_TRUE(FS.get() != nullptr);
+
+ std::error_code EC;
+ checkContents(FS->dir_begin("//root/", EC),
+ {"//root/a", "//root/b"});
+}
+
+TEST_F(VFSFromYAMLTest, DirectoryIterationErrorInVFSLayer) {
+ IntrusiveRefCntPtr<DummyFileSystem> Lower(new DummyFileSystem());
+ Lower->addDirectory("//root/");
+ Lower->addDirectory("//root/foo");
+ Lower->addRegularFile("//root/foo/a");
+ Lower->addRegularFile("//root/foo/b");
+ IntrusiveRefCntPtr<vfs::FileSystem> FS = getFromYAMLString(
+ "{ 'use-external-names': false,\n"
+ " 'roots': [\n"
+ "{\n"
+ " 'type': 'directory',\n"
+ " 'name': '//root/',\n"
+ " 'contents': [ {\n"
+ " 'type': 'file',\n"
+ " 'name': 'bar/a',\n"
+ " 'external-contents': '//root/foo/a'\n"
+ " }\n"
+ " ]\n"
+ "}\n"
+ "]\n"
+ "}",
+ Lower);
+ ASSERT_TRUE(FS.get() != nullptr);
+
+ std::error_code EC;
+ checkContents(FS->dir_begin("//root/foo", EC),
+ {"//root/foo/a", "//root/foo/b"});
+}
More information about the llvm-commits
mailing list