[clang] [clang][deps] Cache `VFS::getRealPath()` (PR #68645)
Jan Svoboda via cfe-commits
cfe-commits at lists.llvm.org
Fri Apr 12 10:34:14 PDT 2024
https://github.com/jansvoboda11 updated https://github.com/llvm/llvm-project/pull/68645
>From 3b89f001adf027b2128c72c7b907b41717ce1e4c Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Mon, 9 Oct 2023 10:14:22 -0700
Subject: [PATCH 01/13] [clang][deps] Cache `VFS::getRealPath()`
---
.../DependencyScanningFilesystem.h | 46 ++++++++++-
.../DependencyScanningFilesystem.cpp | 76 +++++++++++++++++++
2 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 4cd0f958fcff82..467b161fc2f0e3 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -142,6 +142,8 @@ class CachedFileSystemEntry {
CachedFileContents *Contents;
};
+using CachedRealPath = llvm::ErrorOr<std::string>;
+
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
/// underlying real file system, and the scanned preprocessor directives of
/// files.
@@ -168,6 +170,12 @@ class DependencyScanningFilesystemSharedCache {
/// The backing storage for cached contents.
llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;
+ /// Map from filenames to cached real paths.
+ llvm::StringMap<const CachedRealPath *> RealPathsByFilename;
+
+ /// The backing storage for cached real paths.
+ llvm::SpecificBumpPtrAllocator<CachedRealPath> RealPathStorage;
+
/// Returns entry associated with the filename or nullptr if none is found.
const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const;
@@ -194,6 +202,17 @@ class DependencyScanningFilesystemSharedCache {
const CachedFileSystemEntry &
getOrInsertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry);
+
+ /// Returns real path associated with the filename or nullptr if none is
+ /// found.
+ const CachedRealPath *findRealPathByFilename(StringRef Filename) const;
+
+ /// Returns real path associated with the filename if there is some.
+ /// Otherwise, constructs new one with the given one, associates it with the
+ /// filename and returns the result.
+ const CachedRealPath &
+ getOrEmplaceRealPathForFilename(StringRef Filename,
+ llvm::ErrorOr<StringRef> RealPath);
};
DependencyScanningFilesystemSharedCache();
@@ -212,6 +231,8 @@ class DependencyScanningFilesystemSharedCache {
class DependencyScanningFilesystemLocalCache {
llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+ llvm::StringMap<const CachedRealPath *, llvm::BumpPtrAllocator> RealPathCache;
+
public:
/// Returns entry associated with the filename or nullptr if none is found.
const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {
@@ -230,6 +251,26 @@ class DependencyScanningFilesystemLocalCache {
assert(InsertedEntry == &Entry && "entry already present");
return *InsertedEntry;
}
+
+ /// Returns real path associated with the filename or nullptr if none is
+ /// found.
+ const CachedRealPath *findRealPathByFilename(StringRef Filename) const {
+ assert(llvm::sys::path::is_absolute_gnu(Filename));
+ auto It = RealPathCache.find(Filename);
+ return It == RealPathCache.end() ? nullptr : It->getValue();
+ }
+
+ /// Associates the given real path with the filename and returns the given
+ /// entry pointer (for convenience).
+ const CachedRealPath &
+ insertRealPathForFilename(StringRef Filename,
+ const CachedRealPath &RealPath) {
+ assert(llvm::sys::path::is_absolute_gnu(Filename));
+ const auto *InsertedRealPath =
+ RealPathCache.insert({Filename, &RealPath}).first->second;
+ assert(InsertedRealPath == &RealPath && "entry already present");
+ return *InsertedRealPath;
+ }
};
/// Reference to a CachedFileSystemEntry.
@@ -296,6 +337,9 @@ class DependencyScanningWorkerFilesystem
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
openFileForRead(const Twine &Path) override;
+ std::error_code getRealPath(const Twine &Path,
+ SmallVectorImpl<char> &Output) const override;
+
std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
/// Returns entry for the given filename.
@@ -395,7 +439,7 @@ class DependencyScanningWorkerFilesystem
DependencyScanningFilesystemSharedCache &SharedCache;
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
- DependencyScanningFilesystemLocalCache LocalCache;
+ mutable DependencyScanningFilesystemLocalCache LocalCache;
/// The working directory to use for making relative paths absolute before
/// using them for cache lookups.
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index c66780d50fa184..b44db58745d62c 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -162,6 +162,35 @@ DependencyScanningFilesystemSharedCache::CacheShard::
return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
}
+const CachedRealPath *
+DependencyScanningFilesystemSharedCache::CacheShard::findRealPathByFilename(
+ StringRef Filename) const {
+ assert(llvm::sys::path::is_absolute_gnu(Filename));
+ std::lock_guard<std::mutex> LockGuard(CacheLock);
+ auto It = RealPathsByFilename.find(Filename);
+ return It == RealPathsByFilename.end() ? nullptr : It->getValue();
+}
+
+const CachedRealPath &DependencyScanningFilesystemSharedCache::CacheShard::
+ getOrEmplaceRealPathForFilename(StringRef Filename,
+ llvm::ErrorOr<llvm::StringRef> RealPath) {
+ std::lock_guard<std::mutex> LockGuard(CacheLock);
+
+ auto Insertion = RealPathsByFilename.insert({Filename, nullptr});
+ if (Insertion.second) {
+ auto OwnedRealPath = [&]() -> CachedRealPath {
+ if (!RealPath)
+ return RealPath.getError();
+ return RealPath->str();
+ }();
+
+ Insertion.first->second = new (RealPathStorage.Allocate())
+ CachedRealPath(std::move(OwnedRealPath));
+ }
+
+ return *Insertion.first->second;
+}
+
static bool shouldCacheStatFailures(StringRef Filename) {
StringRef Ext = llvm::sys::path::extension(Filename);
if (Ext.empty())
@@ -321,6 +350,53 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
return DepScanFile::create(Result.get());
}
+std::error_code DependencyScanningWorkerFilesystem::getRealPath(
+ const Twine &Path, SmallVectorImpl<char> &Output) const {
+ SmallString<256> OwnedFilename;
+ StringRef OriginalFilename = Path.toStringRef(OwnedFilename);
+
+ SmallString<256> PathBuf;
+ auto FilenameForLookup = tryGetFilenameForLookup(OriginalFilename, PathBuf);
+ if (!FilenameForLookup)
+ return FilenameForLookup.getError();
+
+ auto HandleCachedRealPath =
+ [&Output](const CachedRealPath &RealPath) -> std::error_code {
+ if (!RealPath)
+ return RealPath.getError();
+ Output.assign(RealPath->begin(), RealPath->end());
+ return {};
+ };
+
+ // If we already have the result in local cache, no work required.
+ if (const auto *RealPath =
+ LocalCache.findRealPathByFilename(*FilenameForLookup))
+ return HandleCachedRealPath(*RealPath);
+
+ // If we have the result in the shared cache, cache it locally.
+ auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
+ if (const auto *ShardRealPath =
+ Shard.findRealPathByFilename(*FilenameForLookup)) {
+ const auto &RealPath =
+ LocalCache.insertRealPathForFilename(*FilenameForLookup, *ShardRealPath);
+ return HandleCachedRealPath(RealPath);
+ }
+
+ // If we don't know the real path, compute it...
+ std::error_code EC = getUnderlyingFS().getRealPath(OriginalFilename, Output);
+ llvm::ErrorOr<llvm::StringRef> ComputedRealPath = EC;
+ if (!EC)
+ ComputedRealPath = StringRef{Output.data(), Output.size()};
+
+ // ...and try to write it into the shared cache. In case some other thread won
+ // this race and already wrote its own result there, just adopt it. Write
+ // whatever is in the shared cache into the local one.
+ const auto &RealPath = Shard.getOrEmplaceRealPathForFilename(
+ *FilenameForLookup, ComputedRealPath);
+ return HandleCachedRealPath(
+ LocalCache.insertRealPathForFilename(*FilenameForLookup, RealPath));
+}
+
std::error_code DependencyScanningWorkerFilesystem::setCurrentWorkingDirectory(
const Twine &Path) {
std::error_code EC = ProxyFileSystem::setCurrentWorkingDirectory(Path);
>From b051a3620656501e77769830956531a25c9cf968 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Fri, 13 Oct 2023 13:26:28 -0700
Subject: [PATCH 02/13] Squash real path cache with the existing cache
---
.../DependencyScanningFilesystem.h | 37 +++++++++----------
.../DependencyScanningFilesystem.cpp | 27 +++++++-------
2 files changed, 32 insertions(+), 32 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 467b161fc2f0e3..438081cac0fb8f 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -156,9 +156,11 @@ class DependencyScanningFilesystemSharedCache {
/// The mutex that needs to be locked before mutation of any member.
mutable std::mutex CacheLock;
- /// Map from filenames to cached entries.
- llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>
- EntriesByFilename;
+ /// Map from filenames to cached entries and real paths.
+ llvm::StringMap<
+ std::pair<const CachedFileSystemEntry *, const CachedRealPath *>,
+ llvm::BumpPtrAllocator>
+ CacheByFilename;
/// Map from unique IDs to cached entries.
llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *>
@@ -170,9 +172,6 @@ class DependencyScanningFilesystemSharedCache {
/// The backing storage for cached contents.
llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;
- /// Map from filenames to cached real paths.
- llvm::StringMap<const CachedRealPath *> RealPathsByFilename;
-
/// The backing storage for cached real paths.
llvm::SpecificBumpPtrAllocator<CachedRealPath> RealPathStorage;
@@ -229,16 +228,17 @@ class DependencyScanningFilesystemSharedCache {
/// This class is a local cache, that caches the 'stat' and 'open' calls to the
/// underlying real file system.
class DependencyScanningFilesystemLocalCache {
- llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
-
- llvm::StringMap<const CachedRealPath *, llvm::BumpPtrAllocator> RealPathCache;
+ llvm::StringMap<
+ std::pair<const CachedFileSystemEntry *, const CachedRealPath *>,
+ llvm::BumpPtrAllocator>
+ Cache;
public:
/// Returns entry associated with the filename or nullptr if none is found.
const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {
assert(llvm::sys::path::is_absolute_gnu(Filename));
auto It = Cache.find(Filename);
- return It == Cache.end() ? nullptr : It->getValue();
+ return It == Cache.end() ? nullptr : It->getValue().first;
}
/// Associates the given entry with the filename and returns the given entry
@@ -247,17 +247,17 @@ class DependencyScanningFilesystemLocalCache {
insertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry) {
assert(llvm::sys::path::is_absolute_gnu(Filename));
- const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second;
- assert(InsertedEntry == &Entry && "entry already present");
- return *InsertedEntry;
+ assert(Cache[Filename].first == nullptr && "entry already present");
+ Cache[Filename].first = &Entry;
+ return Entry;
}
/// Returns real path associated with the filename or nullptr if none is
/// found.
const CachedRealPath *findRealPathByFilename(StringRef Filename) const {
assert(llvm::sys::path::is_absolute_gnu(Filename));
- auto It = RealPathCache.find(Filename);
- return It == RealPathCache.end() ? nullptr : It->getValue();
+ auto It = Cache.find(Filename);
+ return It == Cache.end() ? nullptr : It->getValue().second;
}
/// Associates the given real path with the filename and returns the given
@@ -266,10 +266,9 @@ class DependencyScanningFilesystemLocalCache {
insertRealPathForFilename(StringRef Filename,
const CachedRealPath &RealPath) {
assert(llvm::sys::path::is_absolute_gnu(Filename));
- const auto *InsertedRealPath =
- RealPathCache.insert({Filename, &RealPath}).first->second;
- assert(InsertedRealPath == &RealPath && "entry already present");
- return *InsertedRealPath;
+ assert(Cache[Filename].second == nullptr && "entry already present");
+ Cache[Filename].second = &RealPath;
+ return RealPath;
}
};
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index b44db58745d62c..204c0edb9b5eb2 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -113,8 +113,8 @@ DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
StringRef Filename) const {
assert(llvm::sys::path::is_absolute_gnu(Filename));
std::lock_guard<std::mutex> LockGuard(CacheLock);
- auto It = EntriesByFilename.find(Filename);
- return It == EntriesByFilename.end() ? nullptr : It->getValue();
+ auto It = CacheByFilename.find(Filename);
+ return It == CacheByFilename.end() ? nullptr : It->getValue().first;
}
const CachedFileSystemEntry *
@@ -130,11 +130,11 @@ DependencyScanningFilesystemSharedCache::CacheShard::
getOrEmplaceEntryForFilename(StringRef Filename,
llvm::ErrorOr<llvm::vfs::Status> Stat) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- auto Insertion = EntriesByFilename.insert({Filename, nullptr});
- if (Insertion.second)
- Insertion.first->second =
+ const CachedFileSystemEntry *StoredEntry = CacheByFilename[Filename].first;
+ if (!StoredEntry)
+ StoredEntry =
new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
- return *Insertion.first->second;
+ return *StoredEntry;
}
const CachedFileSystemEntry &
@@ -159,7 +159,8 @@ DependencyScanningFilesystemSharedCache::CacheShard::
getOrInsertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
+ CacheByFilename[Filename].first = &Entry;
+ return Entry;
}
const CachedRealPath *
@@ -167,8 +168,8 @@ DependencyScanningFilesystemSharedCache::CacheShard::findRealPathByFilename(
StringRef Filename) const {
assert(llvm::sys::path::is_absolute_gnu(Filename));
std::lock_guard<std::mutex> LockGuard(CacheLock);
- auto It = RealPathsByFilename.find(Filename);
- return It == RealPathsByFilename.end() ? nullptr : It->getValue();
+ auto It = CacheByFilename.find(Filename);
+ return It == CacheByFilename.end() ? nullptr : It->getValue().second;
}
const CachedRealPath &DependencyScanningFilesystemSharedCache::CacheShard::
@@ -176,19 +177,19 @@ const CachedRealPath &DependencyScanningFilesystemSharedCache::CacheShard::
llvm::ErrorOr<llvm::StringRef> RealPath) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- auto Insertion = RealPathsByFilename.insert({Filename, nullptr});
- if (Insertion.second) {
+ const CachedRealPath *StoredRealPath = CacheByFilename[Filename].second;
+ if (!StoredRealPath) {
auto OwnedRealPath = [&]() -> CachedRealPath {
if (!RealPath)
return RealPath.getError();
return RealPath->str();
}();
- Insertion.first->second = new (RealPathStorage.Allocate())
+ StoredRealPath = new (RealPathStorage.Allocate())
CachedRealPath(std::move(OwnedRealPath));
}
- return *Insertion.first->second;
+ return *StoredRealPath;
}
static bool shouldCacheStatFailures(StringRef Filename) {
>From 3c8fbabddb7a4d2c713abe5cd1b7c66b8ad48521 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Wed, 10 Apr 2024 10:21:42 -0700
Subject: [PATCH 03/13] Make `getRealPath()` non-const
---
.../DependencyScanning/DependencyScanningFilesystem.h | 4 ++--
.../DependencyScanning/DependencyScanningFilesystem.cpp | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 438081cac0fb8f..68890039e953a8 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -337,7 +337,7 @@ class DependencyScanningWorkerFilesystem
openFileForRead(const Twine &Path) override;
std::error_code getRealPath(const Twine &Path,
- SmallVectorImpl<char> &Output) const override;
+ SmallVectorImpl<char> &Output) override;
std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
@@ -438,7 +438,7 @@ class DependencyScanningWorkerFilesystem
DependencyScanningFilesystemSharedCache &SharedCache;
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
- mutable DependencyScanningFilesystemLocalCache LocalCache;
+ DependencyScanningFilesystemLocalCache LocalCache;
/// The working directory to use for making relative paths absolute before
/// using them for cache lookups.
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 204c0edb9b5eb2..374fad52be92a1 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -351,8 +351,9 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
return DepScanFile::create(Result.get());
}
-std::error_code DependencyScanningWorkerFilesystem::getRealPath(
- const Twine &Path, SmallVectorImpl<char> &Output) const {
+std::error_code
+DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
+ SmallVectorImpl<char> &Output) {
SmallString<256> OwnedFilename;
StringRef OriginalFilename = Path.toStringRef(OwnedFilename);
>From 8b87e5244387200692608cfd6a31adec75ae3d54 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Wed, 10 Apr 2024 10:50:14 -0700
Subject: [PATCH 04/13] Add test
---
clang/unittests/Tooling/CMakeLists.txt | 1 +
.../DependencyScanningFilesystemTest.cpp | 64 +++++++++++++++++++
2 files changed, 65 insertions(+)
create mode 100644 clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 2ff493ef5fc324..0eb612f8d94987 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -24,6 +24,7 @@ add_clang_unittest(ToolingTests
QualTypeNamesTest.cpp
RangeSelectorTest.cpp
DependencyScanning/DependencyScannerTest.cpp
+ DependencyScanning/DependencyScanningFilesystemTest.cpp
RecursiveASTVisitorTests/Attr.cpp
RecursiveASTVisitorTests/BitfieldInitializer.cpp
RecursiveASTVisitorTests/CallbacksLeaf.cpp
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
new file mode 100644
index 00000000000000..4355decb0afa8a
--- /dev/null
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -0,0 +1,64 @@
+//===- DependencyScanningFilesystemTest.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "gtest/gtest.h"
+
+using namespace clang::tooling::dependencies;
+
+namespace {
+struct InstrumentingFilesystem
+ : llvm::RTTIExtends<InstrumentingFilesystem, llvm::vfs::ProxyFileSystem> {
+ unsigned NumGetRealPathCalls = 0;
+
+ using llvm::RTTIExtends<InstrumentingFilesystem,
+ llvm::vfs::ProxyFileSystem>::RTTIExtends;
+
+ std::error_code getRealPath(const llvm::Twine &Path,
+ llvm::SmallVectorImpl<char> &Output) override {
+ ++NumGetRealPathCalls;
+ return ProxyFileSystem::getRealPath(Path, Output);
+ }
+};
+} // namespace
+
+TEST(DependencyScanningFilesystem, CacheGetRealPath) {
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ InMemoryFS->setCurrentWorkingDirectory("/");
+ InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
+ InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+ auto InstrumentingFS =
+ llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+ DependencyScanningFilesystemSharedCache SharedCache;
+ DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/foo", Result);
+ EXPECT_EQ(Result, "/foo");
+ EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u);
+ }
+
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/foo", Result);
+ EXPECT_EQ(Result, "/foo");
+ EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u); // Cached, no increase.
+ }
+
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/bar", Result);
+ EXPECT_EQ(Result, "/bar");
+ EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u);
+ }
+}
>From d532e3287da9d1a943fe22f38f9fd573e5404003 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Wed, 10 Apr 2024 11:01:14 -0700
Subject: [PATCH 05/13] Formatting
---
.../DependencyScanning/DependencyScanningFilesystem.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 374fad52be92a1..6634e6d85f329b 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -379,8 +379,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
if (const auto *ShardRealPath =
Shard.findRealPathByFilename(*FilenameForLookup)) {
- const auto &RealPath =
- LocalCache.insertRealPathForFilename(*FilenameForLookup, *ShardRealPath);
+ const auto &RealPath = LocalCache.insertRealPathForFilename(
+ *FilenameForLookup, *ShardRealPath);
return HandleCachedRealPath(RealPath);
}
>From 57facb6e2d055d7b47c8f2f51601a83e8bb5863e Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Wed, 10 Apr 2024 11:26:05 -0700
Subject: [PATCH 06/13] Fix propagation to shared cache
---
.../DependencyScanning/DependencyScanningFilesystem.cpp | 2 +-
.../DependencyScanningFilesystemTest.cpp | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 6634e6d85f329b..fe2f178def60be 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -177,7 +177,7 @@ const CachedRealPath &DependencyScanningFilesystemSharedCache::CacheShard::
llvm::ErrorOr<llvm::StringRef> RealPath) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- const CachedRealPath *StoredRealPath = CacheByFilename[Filename].second;
+ const CachedRealPath *&StoredRealPath = CacheByFilename[Filename].second;
if (!StoredRealPath) {
auto OwnedRealPath = [&]() -> CachedRealPath {
if (!RealPath)
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 4355decb0afa8a..e9cb92fd7c9172 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -40,6 +40,7 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
DependencyScanningFilesystemSharedCache SharedCache;
DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+ DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
{
llvm::SmallString<128> Result;
@@ -61,4 +62,11 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
EXPECT_EQ(Result, "/bar");
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u);
}
+
+ {
+ llvm::SmallString<128> Result;
+ DepFS2.getRealPath("/foo", Result);
+ EXPECT_EQ(Result, "/foo");
+ EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u); // Shared cache.
+ }
}
>From f16b2d6aaab8b105913d330500c4b4fb61301778 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Wed, 10 Apr 2024 16:33:35 -0700
Subject: [PATCH 07/13] Remove unnecessary Result checks that fail on Windows
---
.../DependencyScanning/DependencyScanningFilesystemTest.cpp | 4 ----
1 file changed, 4 deletions(-)
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index e9cb92fd7c9172..408e7874ee4f83 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -45,28 +45,24 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
{
llvm::SmallString<128> Result;
DepFS.getRealPath("/foo", Result);
- EXPECT_EQ(Result, "/foo");
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u);
}
{
llvm::SmallString<128> Result;
DepFS.getRealPath("/foo", Result);
- EXPECT_EQ(Result, "/foo");
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u); // Cached, no increase.
}
{
llvm::SmallString<128> Result;
DepFS.getRealPath("/bar", Result);
- EXPECT_EQ(Result, "/bar");
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u);
}
{
llvm::SmallString<128> Result;
DepFS2.getRealPath("/foo", Result);
- EXPECT_EQ(Result, "/foo");
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u); // Shared cache.
}
}
>From fe7b536c61433553a3d1ebb80c2686f8b8110f8e Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 13:28:58 -0700
Subject: [PATCH 08/13] Add "the" to the doc comment
---
.../Tooling/DependencyScanning/DependencyScanningFilesystem.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 68890039e953a8..3b1bf03e113130 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -202,11 +202,11 @@ class DependencyScanningFilesystemSharedCache {
getOrInsertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry);
- /// Returns real path associated with the filename or nullptr if none is
+ /// Returns the real path associated with the filename or nullptr if none is
/// found.
const CachedRealPath *findRealPathByFilename(StringRef Filename) const;
- /// Returns real path associated with the filename if there is some.
+ /// Returns the real path associated with the filename if there is some.
/// Otherwise, constructs new one with the given one, associates it with the
/// filename and returns the result.
const CachedRealPath &
>From 6dd1fa8c18ec034cc4dd24435f1d0e0418b86bba Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 13:29:36 -0700
Subject: [PATCH 09/13] Remove double lookup in local cache for assert builds
---
.../DependencyScanningFilesystem.h | 24 ++++++++++++++-----
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 3b1bf03e113130..8b6f149c7cb266 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -247,9 +247,15 @@ class DependencyScanningFilesystemLocalCache {
insertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry) {
assert(llvm::sys::path::is_absolute_gnu(Filename));
- assert(Cache[Filename].first == nullptr && "entry already present");
- Cache[Filename].first = &Entry;
- return Entry;
+ auto [It, Inserted] = Cache.insert({Filename, {&Entry, nullptr}});
+ auto &[CachedEntry, CachedRealPath] = It->getValue();
+ if (!Inserted) {
+ // The file is already present in the local cache. If we got here, it only
+ // contains the real path. Let's make sure the entry is populated too.
+ assert((!CachedEntry && CachedRealPath) && "entry already present");
+ CachedEntry = &Entry;
+ }
+ return *CachedEntry;
}
/// Returns real path associated with the filename or nullptr if none is
@@ -266,9 +272,15 @@ class DependencyScanningFilesystemLocalCache {
insertRealPathForFilename(StringRef Filename,
const CachedRealPath &RealPath) {
assert(llvm::sys::path::is_absolute_gnu(Filename));
- assert(Cache[Filename].second == nullptr && "entry already present");
- Cache[Filename].second = &RealPath;
- return RealPath;
+ auto [It, Inserted] = Cache.insert({Filename, {nullptr, &RealPath}});
+ auto &[CachedEntry, CachedRealPath] = It->getValue();
+ if (!Inserted) {
+ // The file is already present in the local cache. If we got here, it only
+ // contains the entry. Let's make sure the real path is populated too.
+ assert((!CachedRealPath && CachedEntry) && "real path already present");
+ CachedRealPath = &RealPath;
+ }
+ return *CachedRealPath;
}
};
>From a686c41a2ac0a96092fde98a6fd70a32ef3c51e9 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 13:32:34 -0700
Subject: [PATCH 10/13] Do not replace existing entry in shared cache
---
.../DependencyScanning/DependencyScanningFilesystem.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index fe2f178def60be..6bc36d44c984d7 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -159,8 +159,11 @@ DependencyScanningFilesystemSharedCache::CacheShard::
getOrInsertEntryForFilename(StringRef Filename,
const CachedFileSystemEntry &Entry) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- CacheByFilename[Filename].first = &Entry;
- return Entry;
+ auto [It, Inserted] = CacheByFilename.insert({Filename, {&Entry, nullptr}});
+ auto &[CachedEntry, CachedRealPath] = It->getValue();
+ if (!Inserted || !CachedEntry)
+ CachedEntry = &Entry;
+ return *CachedEntry;
}
const CachedRealPath *
>From 8d7f7d4aad14a700efe65ac9c203994acc41e293 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 13:34:18 -0700
Subject: [PATCH 11/13] Propagate stat failures to shared cache by adding
forgotten reference
---
.../DependencyScanningFilesystem.cpp | 9 +++---
.../DependencyScanningFilesystemTest.cpp | 29 +++++++++++++++++++
2 files changed, 34 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 6bc36d44c984d7..669e0ee33bd224 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -130,11 +130,12 @@ DependencyScanningFilesystemSharedCache::CacheShard::
getOrEmplaceEntryForFilename(StringRef Filename,
llvm::ErrorOr<llvm::vfs::Status> Stat) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- const CachedFileSystemEntry *StoredEntry = CacheByFilename[Filename].first;
- if (!StoredEntry)
- StoredEntry =
+ auto [It, Inserted] = CacheByFilename.insert({Filename, {nullptr, nullptr}});
+ auto &[CachedEntry, CachedRealPath] = It->getValue();
+ if (!CachedEntry)
+ CachedEntry =
new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
- return *StoredEntry;
+ return *CachedEntry;
}
const CachedFileSystemEntry &
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 408e7874ee4f83..a50134ba6beae6 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -16,11 +16,17 @@ using namespace clang::tooling::dependencies;
namespace {
struct InstrumentingFilesystem
: llvm::RTTIExtends<InstrumentingFilesystem, llvm::vfs::ProxyFileSystem> {
+ unsigned NumStatusCalls = 0;
unsigned NumGetRealPathCalls = 0;
using llvm::RTTIExtends<InstrumentingFilesystem,
llvm::vfs::ProxyFileSystem>::RTTIExtends;
+ llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
+ ++NumStatusCalls;
+ return ProxyFileSystem::status(Path);
+ }
+
std::error_code getRealPath(const llvm::Twine &Path,
llvm::SmallVectorImpl<char> &Output) override {
++NumGetRealPathCalls;
@@ -29,6 +35,29 @@ struct InstrumentingFilesystem
};
} // namespace
+TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) {
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+
+ auto InstrumentingFS =
+ llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+ DependencyScanningFilesystemSharedCache SharedCache;
+ DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+ DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+
+ DepFS.status("/foo.c");
+ EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
+
+ DepFS.status("/foo.c");
+ EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u); // Cached, no increase.
+
+ DepFS.status("/bar.c");
+ EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u);
+
+ DepFS2.status("/foo.c");
+ EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u); // Shared cache.
+}
+
TEST(DependencyScanningFilesystem, CacheGetRealPath) {
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory("/");
>From ab3850feb7de848c34e32b71942d283e34deac05 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 13:41:31 -0700
Subject: [PATCH 12/13] NFCI: Use structured binding to align with rest of file
---
.../DependencyScanning/DependencyScanningFilesystem.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 669e0ee33bd224..ff70ecf1d8d2ac 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -143,16 +143,17 @@ DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID(
llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
std::unique_ptr<llvm::MemoryBuffer> Contents) {
std::lock_guard<std::mutex> LockGuard(CacheLock);
- auto Insertion = EntriesByUID.insert({UID, nullptr});
- if (Insertion.second) {
+ auto [It, Inserted] = EntriesByUID.insert({UID, nullptr});
+ auto &CachedEntry = It->getSecond();
+ if (Inserted) {
CachedFileContents *StoredContents = nullptr;
if (Contents)
StoredContents = new (ContentsStorage.Allocate())
CachedFileContents(std::move(Contents));
- Insertion.first->second = new (EntryStorage.Allocate())
+ CachedEntry = new (EntryStorage.Allocate())
CachedFileSystemEntry(std::move(Stat), StoredContents);
}
- return *Insertion.first->second;
+ return *CachedEntry;
}
const CachedFileSystemEntry &
>From fce732e9f60661ef02758d00f4fc21fcb4e82308 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 11 Apr 2024 14:32:10 -0700
Subject: [PATCH 13/13] Add and test extra assertion
---
.../DependencyScanningFilesystem.cpp | 6 +-
.../DependencyScanningFilesystemTest.cpp | 55 +++++++++++++++++++
2 files changed, 60 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index ff70ecf1d8d2ac..84185c931b552c 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -132,9 +132,13 @@ DependencyScanningFilesystemSharedCache::CacheShard::
std::lock_guard<std::mutex> LockGuard(CacheLock);
auto [It, Inserted] = CacheByFilename.insert({Filename, {nullptr, nullptr}});
auto &[CachedEntry, CachedRealPath] = It->getValue();
- if (!CachedEntry)
+ if (!CachedEntry) {
+ // The entry is not present in the shared cache. Either the cache doesn't
+ // know about the file at all, or it only knows about its real path.
+ assert((Inserted || CachedRealPath) && "existing file with empty pair");
CachedEntry =
new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
+ }
return *CachedEntry;
}
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index a50134ba6beae6..50cad72d223e35 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -95,3 +95,58 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u); // Shared cache.
}
}
+
+TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
+ auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+ InMemoryFS->setCurrentWorkingDirectory("/");
+ InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
+ InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+ auto InstrumentingFS =
+ llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+ DependencyScanningFilesystemSharedCache SharedCache;
+ DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+
+ // Success.
+ {
+ DepFS.status("/foo.c");
+
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/foo.c", Result);
+ }
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/bar.c", Result);
+
+ DepFS.status("/bar.c");
+ }
+
+ // Failure.
+ {
+ DepFS.status("/foo.m");
+
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/foo.m", Result);
+ }
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/bar.m", Result);
+
+ DepFS.status("/bar.m");
+ }
+
+ // Failure without caching.
+ {
+ DepFS.status("/foo");
+
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/foo", Result);
+ }
+ {
+ llvm::SmallString<128> Result;
+ DepFS.getRealPath("/bar", Result);
+
+ DepFS.status("/bar");
+ }
+}
More information about the cfe-commits
mailing list