[clang] cf12709 - Revert "[Clang] Give Clang the ability to use a shared stat cache"
Fred Riss via cfe-commits
cfe-commits at lists.llvm.org
Thu Jan 19 14:08:04 PST 2023
Author: Fred Riss
Date: 2023-01-19T14:07:34-08:00
New Revision: cf12709222a4699ff5a4bb257cb891b55b5f6fba
URL: https://github.com/llvm/llvm-project/commit/cf12709222a4699ff5a4bb257cb891b55b5f6fba
DIFF: https://github.com/llvm/llvm-project/commit/cf12709222a4699ff5a4bb257cb891b55b5f6fba.diff
LOG: Revert "[Clang] Give Clang the ability to use a shared stat cache"
This reverts commit c5abe893120b115907376359a5809229a9f9608a.
This reverts commit a033dbbe5c43247b60869b008e67ed86ed230eaa.
This broke the build with -DLLVM_LINK_LLVM_DYLIB=ON. Reverting while I
investigate.
Added:
Modified:
clang/include/clang/Basic/DiagnosticFrontendKinds.td
clang/include/clang/Driver/Options.td
clang/include/clang/Frontend/CompilerInvocation.h
clang/include/clang/Lex/HeaderSearchOptions.h
clang/lib/Frontend/ASTUnit.cpp
clang/lib/Frontend/CompilerInvocation.cpp
clang/test/CMakeLists.txt
clang/tools/CMakeLists.txt
llvm/lib/Support/CMakeLists.txt
llvm/unittests/Support/VirtualFileSystemTest.cpp
Removed:
clang/test/Driver/vfsstatcache.c
clang/test/clang-stat-cache/cache-effects.c
clang/test/clang-stat-cache/errors.test
clang/tools/clang-stat-cache/CMakeLists.txt
clang/tools/clang-stat-cache/clang-stat-cache.cpp
llvm/include/llvm/Support/StatCacheFileSystem.h
llvm/lib/Support/StatCacheFileSystem.cpp
################################################################################
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index e106858688ac7..d0f672ae5a1bd 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -256,11 +256,6 @@ def err_test_module_file_extension_version : Error<
"test module file extension '%0' has
diff erent version (%1.%2) than expected "
"(%3.%4)">;
-def err_missing_vfs_stat_cache_file : Error<
- "stat cache file '%0' not found">, DefaultFatal;
-def err_invalid_vfs_stat_cache : Error<
- "invalid stat cache file '%0'">, DefaultFatal;
-
def err_missing_vfs_overlay_file : Error<
"virtual filesystem overlay file '%0' not found">, DefaultFatal;
def err_invalid_vfs_overlay : Error<
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9334e6319d57b..ba49b335cf287 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3357,8 +3357,6 @@ def iwithsysroot : JoinedOrSeparate<["-"], "iwithsysroot">, Group<clang_i_Group>
HelpText<"Add directory to SYSTEM include search path, "
"absolute paths are relative to -isysroot">, MetaVarName<"<directory>">,
Flags<[CC1Option]>;
-def ivfsstatcache : JoinedOrSeparate<["-"], "ivfsstatcache">, Group<clang_i_Group>, Flags<[CC1Option]>,
- HelpText<"Use the stat data cached in file instead of doing filesystem syscalls. See clang-stat-cache utility.">;
def ivfsoverlay : JoinedOrSeparate<["-"], "ivfsoverlay">, Group<clang_i_Group>, Flags<[CC1Option]>,
HelpText<"Overlay the virtual filesystem described by file over the real file system">;
def imultilib : Separate<["-"], "imultilib">, Group<gfortran_Group>;
diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h
index 9cc6aa5c4d8a1..254f048ed3c7e 100644
--- a/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/clang/include/clang/Frontend/CompilerInvocation.h
@@ -296,7 +296,6 @@ IntrusiveRefCntPtr<llvm::vfs::FileSystem> createVFSFromCompilerInvocation(
IntrusiveRefCntPtr<llvm::vfs::FileSystem>
createVFSFromOverlayFiles(ArrayRef<std::string> VFSOverlayFiles,
- ArrayRef<std::string> VFSStatCacheFiles,
DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS);
diff --git a/clang/include/clang/Lex/HeaderSearchOptions.h b/clang/include/clang/Lex/HeaderSearchOptions.h
index 548f7d4493de4..6436a9b3bde20 100644
--- a/clang/include/clang/Lex/HeaderSearchOptions.h
+++ b/clang/include/clang/Lex/HeaderSearchOptions.h
@@ -181,9 +181,6 @@ class HeaderSearchOptions {
/// of computing the module hash.
llvm::SmallSetVector<llvm::CachedHashString, 16> ModulesIgnoreMacros;
- /// The set of user-provided stat cache files.
- std::vector<std::string> VFSStatCacheFiles;
-
/// The set of user-provided virtual filesystem overlay files.
std::vector<std::string> VFSOverlayFiles;
@@ -253,10 +250,6 @@ class HeaderSearchOptions {
SystemHeaderPrefixes.emplace_back(Prefix, IsSystemHeader);
}
- void AddVFSStatCacheFile(StringRef Name) {
- VFSStatCacheFiles.push_back(std::string(Name));
- }
-
void AddVFSOverlayFile(StringRef Name) {
VFSOverlayFiles.push_back(std::string(Name));
}
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index dbf55d95746e9..3b4f25182ac95 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -574,7 +574,7 @@ class ASTInfoCollector : public ASTReaderListener {
// performs the initialization too late (once both target and language
// options are read).
PP.getFileManager().setVirtualFileSystem(createVFSFromOverlayFiles(
- HSOpts.VFSOverlayFiles, HSOpts.VFSStatCacheFiles, PP.getDiagnostics(),
+ HSOpts.VFSOverlayFiles, PP.getDiagnostics(),
PP.getFileManager().getVirtualFileSystemPtr()));
InitializedHeaderSearchPaths = true;
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index b0ef37fad2227..0bb9c8c83c63b 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -83,7 +83,6 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Regex.h"
-#include "llvm/Support/StatCacheFileSystem.h"
#include "llvm/Support/VersionTuple.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
@@ -3085,9 +3084,6 @@ static void GenerateHeaderSearchArgs(HeaderSearchOptions &Opts,
GenerateArg(Args, Opt, P.Prefix, SA);
}
- for (const std::string &F : Opts.VFSStatCacheFiles)
- GenerateArg(Args, OPT_ivfsstatcache, F, SA);
-
for (const std::string &F : Opts.VFSOverlayFiles)
GenerateArg(Args, OPT_ivfsoverlay, F, SA);
}
@@ -3221,9 +3217,6 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
Opts.AddSystemHeaderPrefix(
A->getValue(), A->getOption().matches(OPT_system_header_prefix));
- for (const auto *A : Args.filtered(OPT_ivfsstatcache))
- Opts.AddVFSStatCacheFile(A->getValue());
-
for (const auto *A : Args.filtered(OPT_ivfsoverlay))
Opts.AddVFSOverlayFile(A->getValue());
@@ -4754,31 +4747,12 @@ clang::createVFSFromCompilerInvocation(
const CompilerInvocation &CI, DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
return createVFSFromOverlayFiles(CI.getHeaderSearchOpts().VFSOverlayFiles,
- CI.getHeaderSearchOpts().VFSStatCacheFiles,
Diags, std::move(BaseFS));
}
IntrusiveRefCntPtr<llvm::vfs::FileSystem> clang::createVFSFromOverlayFiles(
- ArrayRef<std::string> VFSOverlayFiles,
- ArrayRef<std::string> VFSStatCacheFiles, DiagnosticsEngine &Diags,
+ ArrayRef<std::string> VFSOverlayFiles, DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
- for (const auto &File : VFSStatCacheFiles) {
- llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
- BaseFS->getBufferForFile(File);
- if (!Buffer) {
- Diags.Report(diag::err_missing_vfs_stat_cache_file) << File;
- continue;
- }
-
- auto StatCache =
- llvm::vfs::StatCacheFileSystem::create(std::move(*Buffer), BaseFS);
-
- if (errorToBool(StatCache.takeError()))
- Diags.Report(diag::err_invalid_vfs_stat_cache) << File;
- else
- BaseFS = std::move(*StatCache);
- }
-
if (VFSOverlayFiles.empty())
return BaseFS;
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index cd3775b55a559..1d6377b5f2d80 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -71,7 +71,6 @@ list(APPEND CLANG_TEST_DEPS
clang-refactor
clang-
diff
clang-scan-deps
- clang-stat-cache
diagtool
hmaptool
)
diff --git a/clang/test/Driver/vfsstatcache.c b/clang/test/Driver/vfsstatcache.c
deleted file mode 100644
index ec3c279d53d08..0000000000000
--- a/clang/test/Driver/vfsstatcache.c
+++ /dev/null
@@ -1,5 +0,0 @@
-// RUN: %clang -ivfsstatcache foo.h -### %s 2>&1 | FileCheck %s
-// CHECK: "-ivfsstatcache" "foo.h"
-
-// RUN: not %clang -ivfsstatcache foo.h %s 2>&1 | FileCheck -check-prefix=CHECK-MISSING %s
-// CHECK-MISSING: stat cache file 'foo.h' not found
diff --git a/clang/test/clang-stat-cache/cache-effects.c b/clang/test/clang-stat-cache/cache-effects.c
deleted file mode 100644
index bf2e2db447e2a..0000000000000
--- a/clang/test/clang-stat-cache/cache-effects.c
+++ /dev/null
@@ -1,63 +0,0 @@
-#include "foo.h"
-
-// Testing the effects of a cache is tricky, because it's just supposed to speed
-// things up, not change the behavior. In this test, we are using an outdated
-// cache to trick HeaderSearch into finding the wrong module and show that it is
-// being used.
-
-// Clear the module cache.
-// RUN: rm -rf %t
-// RUN: mkdir -p %t/Inputs
-// RUN: mkdir -p %t/Inputs/Foo1
-// RUN: mkdir -p %t/Inputs/Foo2
-// RUN: mkdir -p %t/modules-to-compare
-
-// ===
-// Create a Foo module in the Foo1 direcotry.
-// RUN: echo 'void meow(void);' > %t/Inputs/Foo1/foo.h
-// RUN: echo 'module Foo { header "foo.h" }' > %t/Inputs/Foo1/module.map
-
-// ===
-// Compile the module. Note that the compiler has 2 header search paths:
-// Foo2 and Foo1 in that order. The module has been created in Foo1, and
-// it is the only version available now.
-// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -Rmodule-build %s 2>&1
-// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-before.pcm
-
-// ===
-// Create a stat cache for our inputs directory
-// RUN: clang-stat-cache %t/Inputs -o %t/stat.cache
-
-// ===
-// As a sanity check, re-run the same compilation with the cache and check that
-// the module does not change.
-// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache %s -Rmodule-build 2>&1
-// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm
-
-// RUN:
diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm
-
-// ===
-// Now introduce a
diff erent Foo module in the Foo2 directory which is before
-// Foo1 in the search paths.
-// RUN: echo 'void meow2(void);' > %t/Inputs/Foo2/foo.h
-// RUN: echo 'module Foo { header "foo.h" }' > %t/Inputs/Foo2/module.map
-
-// ===
-// Because we're using the (now-outdated) stat cache, this compilation
-// should still be using the first module. It will not see the new one
-// which is earlier in the search paths.
-// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache -Rmodule-build -Rmodule-import %s 2>&1
-// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm
-
-// RUN:
diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm
-
-// ===
-// Regenerate the stat cache for our Inputs directory
-// RUN: clang-stat-cache -f %t/Inputs -o %t/stat.cache 2>&1
-
-// ===
-// Use the module and now see that we are recompiling the new one.
-// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache -Rmodule-build %s 2>&1
-// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm
-
-// RUN: not
diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm
diff --git a/clang/test/clang-stat-cache/errors.test b/clang/test/clang-stat-cache/errors.test
deleted file mode 100644
index ab73a1348f10e..0000000000000
--- a/clang/test/clang-stat-cache/errors.test
+++ /dev/null
@@ -1,42 +0,0 @@
-RUN: rm -rf %t
-RUN: mkdir -p %t
-
-RUN: not clang-stat-cache %t/not-there -o %t/stat.cache 2>&1 | FileCheck --check-prefix=NO-SUCH-DIR %s
-NO-SUCH-DIR: Failed to stat the target directory: {{[Nn]}}o such file or directory
-
-RUN: not clang-stat-cache %t -o %t/not-there/stat.cache 2>&1 | FileCheck --check-prefix=NO-SUCH-FILE %s
-NO-SUCH-FILE: Failed to open cache file: '{{.*}}': {{[Nn]}}o such file or directory
-
-# Use mixed-case directories to exercise the case insensitive implementation.
-RUN: mkdir -p %t/Dir
-RUN: mkdir -p %t/Dir2
-
-# Try to overwrite a few invalid caches
-RUN: echo "Not a stat cache" > %t/stat.cache
-RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s
-RUN: echo "Not a stat cache, but bigger than the stat cache header" > %t/stat.cache
-RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s
-RUN: echo "STAT. This has the correct MAGIC and is bigger than the header." > %t/stat.cache
-RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s
-
-INVALID-CACHE: The output cache file exists and is not a valid stat cache. Aborting.
-
-# Test the force flag
-RUN: echo "STAT. This has the correct MAGIC and is bigger than the header." > %t/stat.cache
-RUN: clang-stat-cache %t/Dir -f -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE-FORCE %s
-INVALID-CACHE-FORCE: The output cache file exists and is not a valid stat cache. Forced update.
-
-# Generate a valid cache for dir
-RUN: rm %t/stat.cache
-RUN: clang-stat-cache %t/Dir -o %t/stat.cache
-RUN: cp %t/stat.cache %t/stat.cache.save
-
-# Try with same base direcotry but with extraneous separators
-RUN: clang-stat-cache %t/Dir/// -v -o %t/stat.cache | FileCheck --check-prefix=EXTRA-SEP %s
-EXTRA-SEP-NOT: Existing cache has
diff erent directory. Regenerating...
-EXTRA-SEP: Cache up-to-date, exiting
-
-# Rewrite the cache with a
diff erent base directory
-RUN: clang-stat-cache %t/Dir2 -o %t/stat.cache 2>&1 | FileCheck --check-prefix=OTHER-DIR %s
-OTHER-DIR: Existing cache has
diff erent directory. Regenerating...
-
diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt
index 147555d87dfc5..f60db6ef0ba34 100644
--- a/clang/tools/CMakeLists.txt
+++ b/clang/tools/CMakeLists.txt
@@ -15,7 +15,6 @@ add_clang_subdirectory(clang-scan-deps)
if(HAVE_CLANG_REPL_SUPPORT)
add_clang_subdirectory(clang-repl)
endif()
-add_clang_subdirectory(clang-stat-cache)
add_clang_subdirectory(c-index-test)
diff --git a/clang/tools/clang-stat-cache/CMakeLists.txt b/clang/tools/clang-stat-cache/CMakeLists.txt
deleted file mode 100644
index ab93d8b3d0f0e..0000000000000
--- a/clang/tools/clang-stat-cache/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-set(LLVM_LINK_COMPONENTS
- Core
- Support
- )
-
-add_clang_tool(clang-stat-cache
- clang-stat-cache.cpp
- )
-
-if(APPLE)
-set(CLANG_STAT_CACHE_LIB_DEPS
- "-framework CoreServices"
- )
-endif()
-
-clang_target_link_libraries(clang-stat-cache
- PRIVATE
- ${CLANG_STAT_CACHE_LIB_DEPS}
- )
diff --git a/clang/tools/clang-stat-cache/clang-stat-cache.cpp b/clang/tools/clang-stat-cache/clang-stat-cache.cpp
deleted file mode 100644
index 183d0d79fadd8..0000000000000
--- a/clang/tools/clang-stat-cache/clang-stat-cache.cpp
+++ /dev/null
@@ -1,318 +0,0 @@
-//===- clang-stat-cache.cpp -----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/StatCacheFileSystem.h"
-#include "llvm/Support/Timer.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <assert.h>
-
-#ifdef __APPLE__
-#include <CoreServices/CoreServices.h>
-
-#include <sys/mount.h>
-#include <sys/param.h>
-#endif // __APPLE__
-
-// The clang-stat-cache utility creates an on-disk cache for the stat data
-// of a file-system tree which is expected to be immutable during a build.
-
-using namespace llvm;
-using llvm::vfs::StatCacheFileSystem;
-
-cl::OptionCategory StatCacheCategory("clang-stat-cache options");
-
-cl::opt<std::string> OutputFilename("o", cl::Required,
- cl::desc("Specify output filename"),
- cl::value_desc("filename"),
- cl::cat(StatCacheCategory));
-
-cl::opt<std::string> TargetDirectory(cl::Positional, cl::Required,
- cl::value_desc("dirname"),
- cl::cat(StatCacheCategory));
-
-cl::opt<bool> Verbose("v", cl::desc("More verbose output"));
-cl::opt<bool> Force("f", cl::desc("Force cache generation"));
-
-#if __APPLE__
-// Used by checkContentsValidity. See below.
-struct CallbackInfo {
- bool SeenChanges = false;
-};
-
-// Used by checkContentsValidity. See below.
-static void FSEventsCallback(ConstFSEventStreamRef streamRef, void *CtxInfo,
- size_t numEvents, void *eventPaths,
- const FSEventStreamEventFlags *eventFlags,
- const FSEventStreamEventId *eventIds) {
- CallbackInfo *Info = static_cast<CallbackInfo *>(CtxInfo);
- for (size_t i = 0; i < numEvents; ++i) {
- // The kFSEventStreamEventFlagHistoryDone is set on the last 'historical'
- // event passed to the callback. This means it is passed after the callback
- // all the relevant activity between the StartEvent of the stream and the
- // point the stream was created.
- // If the callback didn't see any other event, it means there haven't been
- // any alterations to the target directory hierarchy and the cache contents
- // is still up-to-date.
- if (eventFlags[i] & kFSEventStreamEventFlagHistoryDone) {
- // Let's stop the main queue and go back to our non-queue code.
- CFRunLoopStop(CFRunLoopGetCurrent());
- break;
- }
-
- // If we see any event outisde of the kFSEventStreamEventFlagHistoryDone
- // one, there have been changes to the target directory.
- Info->SeenChanges = true;
- }
-}
-
-// FSEvents-based check for cache contents validity. We store the latest
-// FSEventStreamEventId in the cache as a ValidityToken and check if any
-// file system events affected the base directory since the cache was
-// generated.
-static bool checkContentsValidity(uint64_t &ValidityToken) {
- CFStringRef TargetDir = CFStringCreateWithCStringNoCopy(
- kCFAllocatorDefault, TargetDirectory.c_str(), kCFStringEncodingASCII,
- kCFAllocatorNull);
- CFArrayRef PathsToWatch =
- CFArrayCreate(nullptr, (const void **)&TargetDir, 1, nullptr);
- CallbackInfo Info;
- FSEventStreamContext Ctx = {0, &Info, nullptr, nullptr, nullptr};
- FSEventStreamRef Stream;
- CFAbsoluteTime Latency = 0; // Latency in seconds. Do not wait.
-
- // Start at the latest event stored in the cache.
- FSEventStreamEventId StartEvent = ValidityToken;
- // Update the Validity token with the current latest event.
- ValidityToken = FSEventsGetCurrentEventId();
-
- // Create the stream
- Stream =
- FSEventStreamCreate(NULL, &FSEventsCallback, &Ctx, PathsToWatch,
- StartEvent, Latency, kFSEventStreamCreateFlagNone);
-
- // Associate the stream with the main queue.
- FSEventStreamSetDispatchQueue(Stream, dispatch_get_main_queue());
- // Start the stream (needs the queue to run to do anything).
- if (!FSEventStreamStart(Stream)) {
- errs() << "Failed to create FS event stream. "
- << "Considering the cache up-to-date.\n";
- return true;
- }
-
- // Start the main queue. It will be exited by our callback when it got
- // confirmed it processed all events.
- CFRunLoopRun();
-
- return !Info.SeenChanges;
-}
-
-#else // __APPLE__
-
-// There is no cross-platform way to implement a validity check. If this
-// platform doesn't support it, just consider the cache contents always
-// valid. When that's the case, the tool running cache generation needs
-// to have the knowledge to do it only when needed.
-static bool checkContentsValidity(uint64_t &ValidityToken) { return true; }
-
-#endif // __APPLE__
-
-// Populate Generator with the stat cache data for the filesystem tree
-// rooted at BasePath.
-static std::error_code
-populateHashTable(StringRef BasePath,
- StatCacheFileSystem::StatCacheWriter &Generator) {
- using namespace llvm;
- using namespace sys::fs;
-
- std::error_code ErrorCode;
-
- // Just loop over the target directory using a recursive iterator.
- // This invocation follows symlinks, so we are going to potentially
- // store the status of the same file multiple times with
diff erent
- // names.
- for (recursive_directory_iterator I(BasePath, ErrorCode), E;
- I != E && !ErrorCode; I.increment(ErrorCode)) {
- StringRef Path = I->path();
- sys::fs::file_status s;
- // This can fail (broken symlink) and leave the file_status with
- // its default values. The reader knows this.
- status(Path, s);
-
- Generator.addEntry(Path, s);
- }
-
- return ErrorCode;
-}
-
-static bool checkCacheValid(int FD, raw_fd_ostream &Out,
- uint64_t &ValidityToken) {
- sys::fs::file_status Status;
- auto EC = sys::fs::status(FD, Status);
- if (EC) {
- llvm::errs() << "fstat failed: "
- << llvm::toString(llvm::errorCodeToError(EC)) << "\n";
- return false;
- }
-
- auto Size = Status.getSize();
- if (Size == 0) {
- // New file.
-#ifdef __APPLE__
- // Get the current (global) FSEvent id and use this as ValidityToken.
- ValidityToken = FSEventsGetCurrentEventId();
-#endif
- return false;
- }
-
- auto ErrorOrBuffer = MemoryBuffer::getOpenFile(
- sys::fs::convertFDToNativeFile(FD), OutputFilename, Status.getSize());
-
- // Refuse to write to this cache file if it exists but its contents do
- // not look like a valid cache file.
- StringRef BaseDir;
- bool IsCaseSensitive;
- bool VersionMatch;
- if (auto E = StatCacheFileSystem::validateCacheFile(
- (*ErrorOrBuffer)->getMemBufferRef(), BaseDir, IsCaseSensitive,
- VersionMatch, ValidityToken)) {
- llvm::errs() << "The output cache file exists and is not a valid stat "
- "cache.";
- if (!Force) {
- llvm::errs() << " Aborting.\n";
- exit(1);
- }
-
- consumeError(std::move(E));
- llvm::errs() << " Forced update.\n";
- return false;
- }
-
- if (BaseDir != TargetDirectory &&
- (IsCaseSensitive || !BaseDir.equals_insensitive(TargetDirectory))) {
- llvm::errs() << "Existing cache has
diff erent directory. Regenerating...\n";
- return false;
- }
-
- if (!VersionMatch) {
- llvm::errs()
- << "Exisitng cache has
diff erent version number. Regenerating...\n";
- return false;
- }
-
- // Basic structure checks have passed. Lets see if we can prove that the cache
- // contents are still valid.
- bool IsValid = checkContentsValidity(ValidityToken);
- if (IsValid) {
- // The cache is valid, but we might have gotten an updated ValidityToken.
- // Update the cache with it as clang-stat-cache is just going to exit after
- // returning from this function.
- StatCacheFileSystem::updateValidityToken(Out, ValidityToken);
- }
- return IsValid && !Force;
-}
-
-int main(int argc, char *argv[]) {
- cl::ParseCommandLineOptions(argc, argv);
-
- llvm::SmallString<128> CanonicalDirectory = StringRef(TargetDirectory);
-
- // Remove extraneous separators from the end of the basename.
- while (!CanonicalDirectory.empty() &&
- sys::path::is_separator(CanonicalDirectory.back()))
- CanonicalDirectory.pop_back();
- // Canonicalize separators on Windows
- llvm::sys::path::make_preferred(CanonicalDirectory);
- TargetDirectory = std::string(CanonicalDirectory);
-
- StringRef Dirname(TargetDirectory);
-
- std::error_code EC;
- int FD;
- EC = sys::fs::openFileForReadWrite(
- OutputFilename, FD, llvm::sys::fs::CD_OpenAlways, llvm::sys::fs::OF_None);
- if (EC) {
- llvm::errs() << "Failed to open cache file: "
- << toString(llvm::createFileError(OutputFilename, EC)) << "\n";
- return 1;
- }
-
- raw_fd_ostream Out(FD, /* ShouldClose=*/true);
-
- uint64_t ValidityToken = 0;
- // Check if the cache is valid and up-to-date.
- if (checkCacheValid(FD, Out, ValidityToken)) {
- if (Verbose)
- outs() << "Cache up-to-date, exiting\n";
- return 0;
- }
-
- if (Verbose)
- outs() << "Building a stat cache for '" << TargetDirectory << "' into '"
- << OutputFilename << "'\n";
-
- // Do not generate a cache for NFS. Iterating huge directory hierarchies
- // over NFS will be very slow. Better to let the compiler search only the
- // pieces that it needs than use a cache that takes ages to populate.
- bool IsLocal;
- EC = sys::fs::is_local(Dirname, IsLocal);
- if (EC) {
- errs() << "Failed to stat the target directory: "
- << llvm::toString(llvm::errorCodeToError(EC)) << "\n";
- return 1;
- }
-
- if (!IsLocal && !Force) {
- errs() << "Target directory is not a local filesystem. "
- << "Not populating the cache.\n";
- return 0;
- }
-
- sys::fs::file_status BaseDirStatus;
- if (std::error_code EC = status(Dirname, BaseDirStatus)) {
- errs() << "Failed to stat the target directory: "
- << llvm::toString(llvm::errorCodeToError(EC)) << "\n";
- return 1;
- }
-
- // Check if the filesystem hosting the target directory is case sensitive.
- bool IsCaseSensitive = true;
-#ifdef _PC_CASE_SENSITIVE
- IsCaseSensitive =
- ::pathconf(TargetDirectory.c_str(), _PC_CASE_SENSITIVE) == 1;
-#endif
- StatCacheFileSystem::StatCacheWriter Generator(
- Dirname, BaseDirStatus, IsCaseSensitive, ValidityToken);
-
- // Populate the cache.
- auto startTime = llvm::TimeRecord::getCurrentTime();
- populateHashTable(Dirname, Generator);
- auto duration = llvm::TimeRecord::getCurrentTime();
- duration -= startTime;
-
- if (Verbose)
- errs() << "populateHashTable took: " << duration.getWallTime() << "s\n";
-
- // Write the cache to disk.
- startTime = llvm::TimeRecord::getCurrentTime();
- int Size = Generator.writeStatCache(Out);
- duration = llvm::TimeRecord::getCurrentTime();
- duration -= startTime;
-
- if (Verbose)
- errs() << "writeStatCache took: " << duration.getWallTime() << "s\n";
-
- // We might have opened a pre-exising cache which was bigger.
- llvm::sys::fs::resize_file(FD, Size);
-
- return 0;
-}
diff --git a/llvm/include/llvm/Support/StatCacheFileSystem.h b/llvm/include/llvm/Support/StatCacheFileSystem.h
deleted file mode 100644
index cf2e06768acd2..0000000000000
--- a/llvm/include/llvm/Support/StatCacheFileSystem.h
+++ /dev/null
@@ -1,110 +0,0 @@
-//===- StatCacheFileSystem.h - Status Caching Proxy File System -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_STATCACHEFILESYSTEM_H
-#define LLVM_SUPPORT_STATCACHEFILESYSTEM_H
-
-#include "llvm/Support/VirtualFileSystem.h"
-
-#include <list>
-
-namespace llvm {
-template <typename T> class OnDiskIterableChainedHashTable;
-template <typename T> class OnDiskChainedHashTableGenerator;
-
-namespace vfs {
-
-/// A ProxyFileSystem using cached information for status() rather than going to
-/// the underlying filesystem.
-///
-/// When dealing with a huge tree of (mostly) immutable filesystem content
-/// like an SDK, it can be very costly to ask the underlying filesystem for
-/// `stat` data. Even when caching the `stat`s internally, having many
-/// concurrent Clangs accessing the same tree in a similar way causes
-/// contention. As SDK files are mostly immutable, we can pre-compute the status
-/// information using clang-stat-cache and use that information directly without
-/// accessing the real filesystem until Clang needs to open a file. This can
-/// speed up module verification and HeaderSearch by significant amounts.
-class StatCacheFileSystem : public llvm::vfs::ProxyFileSystem {
- class StatCacheLookupInfo;
- using StatCacheType =
- llvm::OnDiskIterableChainedHashTable<StatCacheLookupInfo>;
-
- class StatCacheGenerationInfo;
- using StatCacheGeneratorType =
- llvm::OnDiskChainedHashTableGenerator<StatCacheGenerationInfo>;
-
- explicit StatCacheFileSystem(std::unique_ptr<llvm::MemoryBuffer> CacheFile,
- IntrusiveRefCntPtr<FileSystem> FS,
- bool IsCaseSensitive);
-
-public:
- /// Create a StatCacheFileSystem from the passed \a CacheBuffer, a
- /// MemoryBuffer representing the contents of the \a CacheFilename file. The
- /// returned filesystem will be overlaid on top of \a FS.
- static Expected<IntrusiveRefCntPtr<StatCacheFileSystem>>
- create(std::unique_ptr<llvm::MemoryBuffer> CacheBuffer,
- IntrusiveRefCntPtr<FileSystem> FS);
-
- /// The status override which will consult the cache if \a Path is in the
- /// cached filesystem tree.
- llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
-
-public:
- /// A helper class to generate stat caches.
- class StatCacheWriter {
- llvm::SmallString<128> BaseDir;
- bool IsCaseSensitive;
- uint64_t ValidityToken;
- StatCacheGeneratorType *Generator;
- std::list<std::string> PathStorage;
-
- public:
- /// Create a StatCacheWriter
- ///
- /// \param BaseDir The base directory for the path. Every filename passed to
- /// addEntry() needs to start with this base directory.
- /// \param Status The status entry for the base directory.
- /// \param IsCaseSensitive Whether the cache is case sensitive.
- /// \param ValidityToken A 64 bits token that gets embedded in the cache and
- /// can be used by generator tools to check for the
- /// cache validity in a platform-specific way.
- StatCacheWriter(StringRef BaseDir, const sys::fs::file_status &Status,
- bool IsCaseSensitive, uint64_t ValidityToken = 0);
- ~StatCacheWriter();
-
- /// Add a cache entry storing \a Status for the file at \a Path.
- void addEntry(StringRef Path, const sys::fs::file_status &Status);
-
- /// Write the cache file to \a Out.
- size_t writeStatCache(raw_fd_ostream &Out);
- };
-
-public:
- /// Validate that the file content in \a Buffer is a valid stat cache file.
- /// \a BaseDir, \a IsCaseSensitive and \a ValidityToken are output parameters
- /// that get populated by this call.
- static Error validateCacheFile(llvm::MemoryBufferRef Buffer,
- StringRef &BaseDir, bool &IsCaseSensitive,
- bool &VersionMatch, uint64_t &ValidityToken);
-
- /// Update the ValidityToken data in \a CacheFile.
- static void updateValidityToken(raw_fd_ostream &CacheFile,
- uint64_t ValidityToken);
-
-private:
- std::unique_ptr<llvm::MemoryBuffer> StatCacheFile;
- llvm::StringRef StatCachePrefix;
- std::unique_ptr<StatCacheType> StatCache;
- bool IsCaseSensitive = true;
-};
-
-} // namespace vfs
-} // namespace llvm
-
-#endif // LLVM_SUPPORT_STATCACHEFILESYSTEM_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 16531c4ec78d2..9b5402fa54f0f 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -212,7 +212,6 @@ add_llvm_component_library(LLVMSupport
SmallVector.cpp
SourceMgr.cpp
SpecialCaseList.cpp
- StatCacheFileSystem.cpp
Statistic.cpp
StringExtras.cpp
StringMap.cpp
diff --git a/llvm/lib/Support/StatCacheFileSystem.cpp b/llvm/lib/Support/StatCacheFileSystem.cpp
deleted file mode 100644
index 4633a0d885d98..0000000000000
--- a/llvm/lib/Support/StatCacheFileSystem.cpp
+++ /dev/null
@@ -1,306 +0,0 @@
-//===- StatCacheFileSystem.cpp - Status Caching Proxy File System ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/StatCacheFileSystem.h"
-
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/OnDiskHashTable.h"
-
-namespace llvm {
-namespace vfs {
-
-class StatCacheFileSystem::StatCacheLookupInfo {
-public:
- typedef StringRef external_key_type;
- typedef StringRef internal_key_type;
- typedef llvm::sys::fs::file_status data_type;
- typedef uint32_t hash_value_type;
- typedef uint32_t offset_type;
-
- static bool EqualKey(const internal_key_type &a, const internal_key_type &b) {
- return a == b;
- }
-
- static hash_value_type ComputeHash(const internal_key_type &a) {
- return hash_value(a);
- }
-
- static std::pair<unsigned, unsigned>
- ReadKeyDataLength(const unsigned char *&d) {
- using namespace llvm::support;
- unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
- unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
- return std::make_pair(KeyLen, DataLen);
- }
-
- static const internal_key_type &GetInternalKey(const external_key_type &x) {
- return x;
- }
-
- static const external_key_type &GetExternalKey(const internal_key_type &x) {
- return x;
- }
-
- static internal_key_type ReadKey(const unsigned char *d, unsigned n) {
- return StringRef((const char *)d, n);
- }
-
- static data_type ReadData(const internal_key_type &k, const unsigned char *d,
- unsigned DataLen) {
- data_type Result;
- memcpy(&Result, d, sizeof(Result));
- return Result;
- }
-};
-
-class StatCacheFileSystem::StatCacheGenerationInfo {
-public:
- typedef StringRef key_type;
- typedef const StringRef &key_type_ref;
- typedef sys::fs::file_status data_type;
- typedef const sys::fs::file_status &data_type_ref;
- typedef uint32_t hash_value_type;
- typedef uint32_t offset_type;
-
- /// Calculate the hash for Key
- static hash_value_type ComputeHash(key_type_ref Key) {
- return static_cast<size_t>(hash_value(Key));
- }
-
- /// Return the lengths, in bytes, of the given Key/Data pair.
- static std::pair<unsigned, unsigned>
- EmitKeyDataLength(raw_ostream &Out, key_type_ref Key, data_type_ref Data) {
- using namespace llvm::support;
- endian::Writer LE(Out, little);
- unsigned KeyLen = Key.size();
- unsigned DataLen = sizeof(Data);
- LE.write<uint16_t>(KeyLen);
- LE.write<uint16_t>(DataLen);
- return std::make_pair(KeyLen, DataLen);
- }
-
- static void EmitKey(raw_ostream &Out, key_type_ref Key, unsigned KeyLen) {
- Out.write(Key.data(), KeyLen);
- }
-
- /// Write Data to Out. DataLen is the length from EmitKeyDataLength.
- static void EmitData(raw_ostream &Out, key_type_ref Key, data_type_ref Data,
- unsigned Len) {
- Out.write((const char *)&Data, Len);
- }
-
- static bool EqualKey(key_type_ref Key1, key_type_ref Key2) {
- return Key1 == Key2;
- }
-};
-
-// The format of the stat cache is (pseudo-code):
-// struct stat_cache {
-// char Magic[4]; // "STAT" or "Stat"
-// uint32_t BucketOffset; // See BucketOffset in OnDiskHashTable.h
-// uint64_t ValidityToken; // Platofrm specific data allowing to check
-// // whether the cache is up-to-date.
-// uint32_t Version; // The stat cache format version.
-// char BaseDir[N]; // Zero terminated path to the base directory
-// < OnDiskHashtable Data > // Data for the has table. The keys are the
-// // relative paths under BaseDir. The data is
-// // llvm::sys::fs::file_status structures.
-// };
-
-#define MAGIC_CASE_SENSITIVE "Stat"
-#define MAGIC_CASE_INSENSITIVE "STAT"
-#define STAT_CACHE_VERSION 1
-
-namespace {
-struct StatCacheHeader {
- char Magic[4];
- uint32_t BucketOffset;
- uint64_t ValidityToken;
- uint32_t Version;
- char BaseDir[1];
-};
-} // namespace
-
-StatCacheFileSystem::StatCacheFileSystem(
- std::unique_ptr<MemoryBuffer> CacheFile, IntrusiveRefCntPtr<FileSystem> FS,
- bool IsCaseSensitive)
- : ProxyFileSystem(std::move(FS)), StatCacheFile(std::move(CacheFile)),
- IsCaseSensitive(IsCaseSensitive) {
- const char *CacheFileStart = StatCacheFile->getBufferStart();
- auto *Header = reinterpret_cast<const StatCacheHeader *>(CacheFileStart);
-
- uint32_t BucketOffset = Header->BucketOffset;
- StatCachePrefix = StringRef(Header->BaseDir);
- // HashTableStart points at the beginning of the data emitted by the
- // OnDiskHashTable.
- const unsigned char *HashTableStart = (const unsigned char *)CacheFileStart +
- StatCachePrefix.size() +
- sizeof(StatCacheHeader);
- StatCache.reset(StatCacheType::Create(
- (const unsigned char *)CacheFileStart + BucketOffset, HashTableStart,
- (const unsigned char *)CacheFileStart));
-}
-
-Expected<IntrusiveRefCntPtr<StatCacheFileSystem>>
-StatCacheFileSystem::create(std::unique_ptr<MemoryBuffer> CacheBuffer,
- IntrusiveRefCntPtr<FileSystem> FS) {
- StringRef BaseDir;
- bool IsCaseSensitive;
- bool VersionMatch;
- uint64_t ValidityToken;
- if (auto E = validateCacheFile(*CacheBuffer, BaseDir, IsCaseSensitive,
- VersionMatch, ValidityToken))
- return std::move(E);
- if (!VersionMatch) {
- return createStringError(inconvertibleErrorCode(),
- CacheBuffer->getBufferIdentifier() +
- ": Mismatched cache file version");
- }
- return new StatCacheFileSystem(std::move(CacheBuffer), FS, IsCaseSensitive);
-}
-
-ErrorOr<Status> StatCacheFileSystem::status(const Twine &Path) {
- SmallString<180> StringPath;
- Path.toVector(StringPath);
- // If the cache is not case sensitive, do all operations on lower-cased paths.
- if (!IsCaseSensitive)
- std::transform(StringPath.begin(), StringPath.end(), StringPath.begin(),
- toLower);
-
- // Canonicalize the path. This removes single dot path components,
- // but it also gets rid of repeating separators.
- llvm::sys::path::remove_dots(StringPath);
-
- // If on Windows, canonicalize separators.
- llvm::sys::path::make_preferred(StringPath);
-
- // Check if the requested path falls into the cache.
- StringRef SuffixPath(StringPath);
- if (!SuffixPath.consume_front(StatCachePrefix))
- return ProxyFileSystem::status(Path);
-
- auto It = StatCache->find(SuffixPath);
- if (It == StatCache->end()) {
- // We didn't find the file in the cache even though it started with the
- // cache prefix. It could be that the file doesn't exist, or the spelling
- // the path is
diff erent. `remove_dots` canonicalizes the path by removing
- // `.` and excess separators, but leaves `..` since it isn't semantically
- // preserving to remove them in the presence of symlinks. If the path
- // does not contain '..' we can safely say it doesn't exist.
- if (std::find(sys::path::begin(SuffixPath), sys::path::end(SuffixPath),
- "..") == sys::path::end(SuffixPath)) {
- return llvm::errc::no_such_file_or_directory;
- }
- return ProxyFileSystem::status(Path);
- }
-
- // clang-stat-cache will record entries for broken symlnks with a default-
- // constructed Status. This will have a default-constructed UinqueID.
- if ((*It).getUniqueID() == llvm::sys::fs::UniqueID())
- return llvm::errc::no_such_file_or_directory;
-
- return llvm::vfs::Status::copyWithNewName(*It, Path);
-}
-
-StatCacheFileSystem::StatCacheWriter::StatCacheWriter(
- StringRef BaseDir, const sys::fs::file_status &Status, bool IsCaseSensitive,
- uint64_t ValidityToken)
- : BaseDir(IsCaseSensitive ? BaseDir.str() : BaseDir.lower()),
- IsCaseSensitive(IsCaseSensitive), ValidityToken(ValidityToken),
- Generator(new StatCacheGeneratorType()) {
- addEntry(BaseDir, Status);
- // If on Windows, canonicalize separators.
- llvm::sys::path::make_preferred(this->BaseDir);
-}
-
-StatCacheFileSystem::StatCacheWriter::~StatCacheWriter() { delete Generator; }
-
-void StatCacheFileSystem::StatCacheWriter::addEntry(
- StringRef Path, const sys::fs::file_status &Status) {
- llvm::SmallString<128> StoredPath;
-
-#if defined(_WIN32)
- StoredPath = Path;
- llvm::sys::path::make_preferred(StoredPath);
- Path = StoredPath;
-#endif
-
- if (!IsCaseSensitive) {
- StoredPath = Path.lower();
- Path = StoredPath;
- }
-
- LLVM_ATTRIBUTE_UNUSED bool Consumed = Path.consume_front(BaseDir);
- assert(Consumed && "Path does not start with expected prefix.");
-
- PathStorage.emplace_back(Path.str());
- Generator->insert(PathStorage.back(), Status);
-}
-
-size_t
-StatCacheFileSystem::StatCacheWriter::writeStatCache(raw_fd_ostream &Out) {
- const uint32_t Version = STAT_CACHE_VERSION;
- // Magic value.
- if (IsCaseSensitive)
- Out.write(MAGIC_CASE_SENSITIVE, 4);
- else
- Out.write(MAGIC_CASE_INSENSITIVE, 4);
- // Placeholder for BucketOffset, filled in below.
- Out.write("\0\0\0\0", 4);
- // Write out the validity token.
- Out.write((const char *)&ValidityToken, sizeof(ValidityToken));
- // Write out the version.
- Out.write((const char *)&Version, sizeof(Version));
- // Write out the base directory for the cache.
- Out.write(BaseDir.c_str(), BaseDir.size() + 1);
- // Write out the hashtable data.
- uint32_t BucketOffset = Generator->Emit(Out);
- int Size = Out.tell();
- // Move back to right after the Magic to insert BucketOffset
- Out.seek(4);
- Out.write((const char *)&BucketOffset, sizeof(BucketOffset));
- return Size;
-}
-
-Error StatCacheFileSystem::validateCacheFile(MemoryBufferRef Buffer,
- StringRef &BaseDir,
- bool &IsCaseSensitive,
- bool &VersionMatch,
- uint64_t &ValidityToken) {
- auto *Header =
- reinterpret_cast<const StatCacheHeader *>(Buffer.getBufferStart());
- if (Buffer.getBufferSize() < sizeof(StatCacheHeader) ||
- (memcmp(Header->Magic, MAGIC_CASE_INSENSITIVE, sizeof(Header->Magic)) &&
- memcmp(Header->Magic, MAGIC_CASE_SENSITIVE, sizeof(Header->Magic))) ||
- Header->BucketOffset > Buffer.getBufferSize())
- return createStringError(inconvertibleErrorCode(), "Invalid cache file");
-
- auto PathLen =
- strnlen(Header->BaseDir,
- Buffer.getBufferSize() - offsetof(StatCacheHeader, BaseDir));
- if (Header->BaseDir[PathLen] != 0)
- return createStringError(inconvertibleErrorCode(), "Invalid cache file");
-
- IsCaseSensitive = Header->Magic[1] == MAGIC_CASE_SENSITIVE[1];
- VersionMatch = Header->Version == STAT_CACHE_VERSION;
- BaseDir = StringRef(Header->BaseDir, PathLen);
- ValidityToken = Header->ValidityToken;
-
- return ErrorSuccess();
-}
-
-void StatCacheFileSystem::updateValidityToken(raw_fd_ostream &CacheFile,
- uint64_t ValidityToken) {
- CacheFile.pwrite(reinterpret_cast<char *>(&ValidityToken),
- sizeof(ValidityToken),
- offsetof(StatCacheHeader, ValidityToken));
-}
-
-} // namespace vfs
-} // namespace llvm
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 89fd0aac17f2f..242bb76865b2c 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -14,11 +14,9 @@
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/StatCacheFileSystem.h"
#include "llvm/Testing/Support/SupportHelpers.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
-#include <list>
#include <map>
#include <string>
@@ -3230,306 +3228,3 @@ TEST(RedirectingFileSystemTest, PrintOutput) {
" DummyFileSystem (RecursiveContents)\n",
Output);
}
-
-class StatCacheFileSystemTest : public ::testing::Test {
-public:
- void SetUp() override {}
-
- template <typename StringCollection>
- void createStatCacheFileSystem(
- StringRef OutputFile, StringRef BaseDir, bool IsCaseSensitive,
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> &Result,
- StringCollection &Filenames,
- IntrusiveRefCntPtr<vfs::FileSystem> Lower = new ErrorDummyFileSystem(),
- uint64_t ValidityToken = 0) {
- sys::fs::file_status s;
- status(BaseDir, s);
- vfs::StatCacheFileSystem::StatCacheWriter Generator(
- BaseDir, s, IsCaseSensitive, ValidityToken);
- std::error_code ErrorCode;
-
- Result.reset();
-
- // Base path should be present in the stat cache.
- Filenames.push_back(std::string(BaseDir));
-
- for (sys::fs::recursive_directory_iterator I(BaseDir, ErrorCode), E;
- I != E && !ErrorCode; I.increment(ErrorCode)) {
- Filenames.push_back(I->path());
- StringRef Path(Filenames.back().c_str());
- status(Path, s);
- Generator.addEntry(Path, s);
- }
-
- {
- raw_fd_ostream StatCacheFile(OutputFile, ErrorCode);
- ASSERT_FALSE(ErrorCode);
- Generator.writeStatCache(StatCacheFile);
- }
-
- loadCacheFile(OutputFile, ValidityToken, Lower, Result);
- }
-
- void loadCacheFile(StringRef OutputFile, uint64_t ExpectedValidityToken,
- IntrusiveRefCntPtr<vfs::FileSystem> Lower,
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> &Result) {
- auto ErrorOrBuffer = MemoryBuffer::getFile(OutputFile);
- EXPECT_TRUE(ErrorOrBuffer);
- StringRef CacheBaseDir;
- bool IsCaseSensitive;
- bool VersionMatch;
- uint64_t FileValidityToken;
- auto E = vfs::StatCacheFileSystem::validateCacheFile(
- (*ErrorOrBuffer)->getMemBufferRef(), CacheBaseDir, IsCaseSensitive,
- VersionMatch, FileValidityToken);
- ASSERT_FALSE(E);
- EXPECT_TRUE(VersionMatch);
- EXPECT_EQ(FileValidityToken, ExpectedValidityToken);
- auto ExpectedCache =
- vfs::StatCacheFileSystem::create(std::move(*ErrorOrBuffer), Lower);
- ASSERT_FALSE(ExpectedCache.takeError());
- Result = *ExpectedCache;
- }
-
- template <typename StringCollection>
- void
- compareStatCacheToRealFS(IntrusiveRefCntPtr<vfs::StatCacheFileSystem> CacheFS,
- const StringCollection &Files) {
- IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
-
- for (auto &File : Files) {
- auto ErrorOrStatus1 = RealFS->status(File);
- auto ErrorOrStatus2 = CacheFS->status(File);
-
- EXPECT_EQ((bool)ErrorOrStatus1, (bool)ErrorOrStatus2);
- if (!ErrorOrStatus1 || !ErrorOrStatus2)
- continue;
-
- vfs::Status s1 = *ErrorOrStatus1, s2 = *ErrorOrStatus2;
- EXPECT_EQ(s1.getName(), s2.getName());
- EXPECT_EQ(s1.getType(), s2.getType());
- EXPECT_EQ(s1.getPermissions(), s2.getPermissions());
- EXPECT_EQ(s1.getLastModificationTime(), s2.getLastModificationTime());
- EXPECT_EQ(s1.getUniqueID(), s2.getUniqueID());
- EXPECT_EQ(s1.getUser(), s2.getUser());
- EXPECT_EQ(s1.getGroup(), s2.getGroup());
- EXPECT_EQ(s1.getSize(), s2.getSize());
- }
- }
-};
-
-TEST_F(StatCacheFileSystemTest, Basic) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempFile _ab(TestDirectory.path("a/b"));
- TempDir _ac(TestDirectory.path("a/c"));
- TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents");
- TempFile _ace(TestDirectory.path("a/c/e"));
- TempFile _acf(TestDirectory.path("a/c/f"), "", "More dummy contents");
- TempDir _ag(TestDirectory.path("a/g"));
- TempFile _agh(TestDirectory.path("a/g/h"));
-
- StringRef BaseDir(_a.path());
-
- SmallVector<std::string, 10> Filenames;
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS,
- Filenames);
- ASSERT_TRUE(StatCacheFS);
- compareStatCacheToRealFS(StatCacheFS, Filenames);
-}
-
-TEST_F(StatCacheFileSystemTest, CaseSensitivity) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempDir _ac(TestDirectory.path("a/c"));
- TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents");
- TempDir _b(TestDirectory.path("B"));
- TempDir _bc(TestDirectory.path("B/c"));
- TempFile _bcd(TestDirectory.path("B/c/D"), "", "Dummy contents");
-
- StringRef BaseDir(TestDirectory.path());
- SmallVector<std::string, 10> Filenames;
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS,
- Filenames);
- ASSERT_TRUE(StatCacheFS);
-
- auto ErrorOrStatus = StatCacheFS->status(_acd.path());
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(_bcd.path());
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/C/d"));
- EXPECT_FALSE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("A/C/d"));
- EXPECT_FALSE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/c/D"));
- EXPECT_FALSE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/c/d"));
- EXPECT_FALSE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/C/d"));
- EXPECT_FALSE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("B/C/D"));
- EXPECT_FALSE(ErrorOrStatus);
-
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ false, StatCacheFS,
- Filenames);
- ASSERT_TRUE(StatCacheFS);
- ErrorOrStatus = StatCacheFS->status(_acd.path());
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(_bcd.path());
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/C/d"));
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("A/C/d"));
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/c/D"));
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/c/d"));
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/C/d"));
- EXPECT_TRUE(ErrorOrStatus);
- ErrorOrStatus = StatCacheFS->status(TestDirectory.path("B/C/D"));
- EXPECT_TRUE(ErrorOrStatus);
-}
-
-TEST_F(StatCacheFileSystemTest, DotDot) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempDir _ab(TestDirectory.path("a/b"));
- TempFile _abd(TestDirectory.path("a/b/d"));
- TempDir _ac(TestDirectory.path("a/c"));
- TempFile _acd(TestDirectory.path("a/c/d"));
-
- StringRef BaseDir(_a.path());
- SmallVector<std::string, 10> Filenames;
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- auto RealFS = vfs::getRealFileSystem();
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS, Filenames,
- RealFS);
- ASSERT_TRUE(StatCacheFS);
-
- // Create a file in the cached prefix after the cache was created.
- TempFile _abe(TestDirectory.path("a/b/e"));
- // Verify the cache is kicking in.
- ASSERT_FALSE(StatCacheFS->status(_abe.path()));
- // We can access the new file using a ".." because the StatCache will
- // just pass that request to the FileSystem below it.
- const SmallString<128> PathsToTest[] = {
- TestDirectory.path("a/b/../e"),
- TestDirectory.path("a/b/../c/d"),
- TestDirectory.path("a/b/.."),
- };
- compareStatCacheToRealFS(StatCacheFS, PathsToTest);
-}
-
-#ifdef LLVM_ON_UNIX
-TEST_F(StatCacheFileSystemTest, Links) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempLink _ab("d", TestDirectory.path("a/b"));
- TempFile _ac(TestDirectory.path("a/c"));
- TempDir _ad(TestDirectory.path("a/d"));
- TempFile _add(TestDirectory.path("a/d/d"), "", "Dummy contents");
- TempFile _ade(TestDirectory.path("a/d/e"));
- TempFile _adf(TestDirectory.path("a/d/f"), "", "More dummy contents");
- TempLink _adg(_ad.path(), TestDirectory.path("a/d/g"));
- TempDir _ah(TestDirectory.path("a/h"));
- TempLink _ahi(_ad.path(), TestDirectory.path("a/h/i"));
- TempLink _ahj("no_such_file", TestDirectory.path("a/h/j"));
-
- StringRef BaseDir(_a.path());
-
- SmallVector<std::string, 10> Filenames;
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS,
- Filenames);
- ASSERT_TRUE(StatCacheFS);
- EXPECT_NE(std::find(Filenames.begin(), Filenames.end(),
- TestDirectory.path("a/d/g/g")),
- Filenames.end());
- EXPECT_NE(std::find(Filenames.begin(), Filenames.end(),
- TestDirectory.path("a/b/e")),
- Filenames.end());
- EXPECT_NE(std::find(Filenames.begin(), Filenames.end(),
- TestDirectory.path("a/h/i/f")),
- Filenames.end());
- EXPECT_NE(std::find(Filenames.begin(), Filenames.end(),
- TestDirectory.path("a/h/j")),
- Filenames.end());
- compareStatCacheToRealFS(StatCacheFS, Filenames);
-
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS, Filenames,
- vfs::getRealFileSystem());
- const SmallString<128> PathsToTest[] = {
- TestDirectory.path("a/h/i/../c"),
- TestDirectory.path("a/b/../d"),
- TestDirectory.path("a/g/g/../c"),
- TestDirectory.path("a/b/.."),
- };
- compareStatCacheToRealFS(StatCacheFS, PathsToTest);
-}
-#endif
-
-TEST_F(StatCacheFileSystemTest, Canonical) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempFile _ab(TestDirectory.path("a/b"));
- TempDir _ac(TestDirectory.path("a/c"));
- TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents");
-
- StringRef BaseDir(_a.path());
- SmallVector<std::string, 10> Filenames;
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS,
- Filenames);
- ASSERT_TRUE(StatCacheFS);
-
- const SmallString<128> PathsToTest[] = {
- TestDirectory.path("./a/b"), TestDirectory.path("a//./b"),
- TestDirectory.path("a///b"), TestDirectory.path("a//c//d"),
- TestDirectory.path("a//c/./d"), TestDirectory.path("a/./././b"),
- TestDirectory.path("a/.//.//.//b"),
- };
- compareStatCacheToRealFS(StatCacheFS, PathsToTest);
-}
-
-TEST_F(StatCacheFileSystemTest, ValidityToken) {
- TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
- TempDir _a(TestDirectory.path("a"));
- TempFile _ab(TestDirectory.path("a/b"));
- TempDir _ac(TestDirectory.path("a/c"));
- TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents");
-
- StringRef BaseDir(_a.path());
- IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS;
- {
- SmallVector<std::string, 10> Filenames;
- uint64_t ValidityToken = 0x1234567890abcfef;
- createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir,
- /* IsCaseSensitive= */ true, StatCacheFS,
- Filenames, new DummyFileSystem(), ValidityToken);
- ASSERT_TRUE(StatCacheFS);
- }
-
- uint64_t UpdatedValidityToken = 0xabcdef0123456789;
- {
- std::error_code EC;
- raw_fd_ostream CacheFile(TestDirectory.path("stat.cache"), EC,
- sys::fs::CD_OpenAlways);
- ASSERT_FALSE(EC);
- vfs::StatCacheFileSystem::updateValidityToken(CacheFile,
- UpdatedValidityToken);
- }
-
- loadCacheFile(TestDirectory.path("stat.cache"), UpdatedValidityToken,
- new DummyFileSystem(), StatCacheFS);
- EXPECT_TRUE(StatCacheFS);
-}
More information about the cfe-commits
mailing list