r177918 - <rdar://problem/13434605> Periodically prune the module cache so that it does not grow forever.
Douglas Gregor
dgregor at apple.com
Mon Mar 25 14:19:16 PDT 2013
Author: dgregor
Date: Mon Mar 25 16:19:16 2013
New Revision: 177918
URL: http://llvm.org/viewvc/llvm-project?rev=177918&view=rev
Log:
<rdar://problem/13434605> Periodically prune the module cache so that it does not grow forever.
Added:
cfe/trunk/test/Modules/prune.m
Modified:
cfe/trunk/docs/Modules.rst
cfe/trunk/include/clang/Driver/Options.td
cfe/trunk/include/clang/Lex/HeaderSearchOptions.h
cfe/trunk/lib/Driver/Tools.cpp
cfe/trunk/lib/Frontend/CompilerInstance.cpp
cfe/trunk/lib/Frontend/CompilerInvocation.cpp
Modified: cfe/trunk/docs/Modules.rst
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/docs/Modules.rst?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/docs/Modules.rst (original)
+++ cfe/trunk/docs/Modules.rst Mon Mar 25 16:19:16 2013
@@ -174,6 +174,12 @@ Command-line parameters
``-fmodules-ignore-macro=macroname``
Instruct modules to ignore the named macro when selecting an appropriate module variant. Use this for macros defined on the command line that don't affect how modules are built, to improve sharing of compiled module files.
+``-fmodules-prune-interval=seconds``
+ Specify the minimum delay (in seconds) between attempts to prune the module cache. Module cache pruning attempts to clear out old, unused module files so that the module cache itself does not grow without bound. The default delay is large (604,800 seconds, or 7 days) because this is an expensive operation. Set this value to 0 to turn off pruning.
+
+``-fmodules-prune-after=seconds``
+ Specify the minimum time (in seconds) for which a file in the module cache must be unused (according to access time) before module pruning will remove it. The default delay is large (2,678,400 seconds, or 31 days) to avoid excessive module rebuilding.
+
Module Map Language
===================
Modified: cfe/trunk/include/clang/Driver/Options.td
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/Options.td?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/include/clang/Driver/Options.td (original)
+++ cfe/trunk/include/clang/Driver/Options.td Mon Mar 25 16:19:16 2013
@@ -499,6 +499,12 @@ def fdelayed_template_parsing : Flag<["-
def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
Flags<[NoForward,CC1Option]>, MetaVarName<"<directory>">,
HelpText<"Specify the module cache path">;
+def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i_Group>,
+ Flags<[CC1Option]>, MetaVarName<"<seconds>">,
+ HelpText<"Specify the interval (in seconds) between attempts to prune the module cache">;
+def fmodules_prune_after : Joined<["-"], "fmodules-prune-after=">, Group<i_Group>,
+ Flags<[CC1Option]>, MetaVarName<"<seconds>">,
+ HelpText<"Specify the interval (in seconds) after which a module file will be considered unused">;
def fmodules : Flag <["-"], "fmodules">, Group<f_Group>, Flags<[NoForward,CC1Option]>,
HelpText<"Enable the 'modules' language feature">;
def fmodules_autolink : Flag <["-"], "fmodules-autolink">, Group<f_Group>, Flags<[NoForward,CC1Option]>,
Modified: cfe/trunk/include/clang/Lex/HeaderSearchOptions.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/HeaderSearchOptions.h?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/HeaderSearchOptions.h (original)
+++ cfe/trunk/include/clang/Lex/HeaderSearchOptions.h Mon Mar 25 16:19:16 2013
@@ -95,6 +95,24 @@ public:
/// Note: Only used for testing!
unsigned DisableModuleHash : 1;
+ /// \brief The interval (in seconds) between pruning operations.
+ ///
+ /// This operation is expensive, because it requires Clang to walk through
+ /// the directory structure of the module cache, stat()'ing and removing
+ /// files.
+ ///
+ /// The default value is large, e.g., the operation runs once a week.
+ unsigned ModuleCachePruneInterval;
+
+ /// \brief The time (in seconds) after which an unused module file will be
+ /// considered unused and will, therefore, be pruned.
+ ///
+ /// When the module cache is pruned, any module file that has not been
+ /// accessed in this many seconds will be removed. The default value is
+ /// large, e.g., a month, to avoid forcing infrequently-used modules to be
+ /// regenerated often.
+ unsigned ModuleCachePruneAfter;
+
/// \brief The set of macro names that should be ignored for the purposes
/// of computing the module hash.
llvm::SetVector<std::string> ModulesIgnoreMacros;
@@ -116,7 +134,10 @@ public:
public:
HeaderSearchOptions(StringRef _Sysroot = "/")
- : Sysroot(_Sysroot), DisableModuleHash(0), UseBuiltinIncludes(true),
+ : Sysroot(_Sysroot), DisableModuleHash(0),
+ ModuleCachePruneInterval(7*24*60*60),
+ ModuleCachePruneAfter(31*24*60*60),
+ UseBuiltinIncludes(true),
UseStandardSystemIncludes(true), UseStandardCXXIncludes(true),
UseLibcxx(false), Verbose(false) {}
Modified: cfe/trunk/lib/Driver/Tools.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/lib/Driver/Tools.cpp (original)
+++ cfe/trunk/lib/Driver/Tools.cpp Mon Mar 25 16:19:16 2013
@@ -2827,6 +2827,8 @@ void Clang::ConstructJob(Compilation &C,
// Pass through all -fmodules-ignore-macro arguments.
Args.AddAllArgs(CmdArgs, options::OPT_fmodules_ignore_macro);
+ Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_interval);
+ Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_after);
// -fmodules-autolink (on by default when modules is enabled) automatically
// links against libraries for imported modules. This requires the
Modified: cfe/trunk/lib/Frontend/CompilerInstance.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInstance.cpp?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/CompilerInstance.cpp (original)
+++ cfe/trunk/lib/Frontend/CompilerInstance.cpp Mon Mar 25 16:19:16 2013
@@ -44,6 +44,8 @@
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/system_error.h"
+#include <sys/stat.h>
+#include <sys/time.h>
using namespace clang;
@@ -996,6 +998,97 @@ static void checkConfigMacro(Preprocesso
<< false;
}
+/// \brief Write a new timestamp file with the given path.
+static void writeTimestampFile(StringRef TimestampFile) {
+ std::string ErrorInfo;
+ llvm::raw_fd_ostream Out(TimestampFile.str().c_str(), ErrorInfo,
+ llvm::raw_fd_ostream::F_Binary);
+}
+
+/// \brief Prune the module cache of modules that haven't been accessed in
+/// a long time.
+static void pruneModuleCache(const HeaderSearchOptions &HSOpts) {
+ struct stat StatBuf;
+ llvm::SmallString<128> TimestampFile;
+ TimestampFile = HSOpts.ModuleCachePath;
+ llvm::sys::path::append(TimestampFile, "modules.timestamp");
+
+ // Try to stat() the timestamp file.
+ if (::stat(TimestampFile.c_str(), &StatBuf)) {
+ // If the timestamp file wasn't there, create one now.
+ if (errno == ENOENT) {
+ writeTimestampFile(TimestampFile);
+ }
+ return;
+ }
+
+ // Check whether the time stamp is older than our pruning interval.
+ // If not, do nothing.
+ time_t TimeStampModTime = StatBuf.st_mtime;
+ time_t CurrentTime = time(0);
+ if (CurrentTime - TimeStampModTime <= HSOpts.ModuleCachePruneInterval) {
+ return;
+ }
+
+ // Write a new timestamp file so that nobody else attempts to prune.
+ // There is a benign race condition here, if two Clang instances happen to
+ // notice at the same time that the timestamp is out-of-date.
+ writeTimestampFile(TimestampFile);
+
+ // Walk the entire module cache, looking for unused module files and module
+ // indices.
+ llvm::error_code EC;
+ SmallString<128> ModuleCachePathNative;
+ llvm::sys::path::native(HSOpts.ModuleCachePath, ModuleCachePathNative);
+ for (llvm::sys::fs::directory_iterator
+ Dir(ModuleCachePathNative.str(), EC), DirEnd;
+ Dir != DirEnd && !EC; Dir.increment(EC)) {
+ // If we don't have a directory, there's nothing to look into.
+ bool IsDirectory;
+ if (llvm::sys::fs::is_directory(Dir->path(), IsDirectory) || !IsDirectory)
+ continue;
+
+ // Walk all of the files within this directory.
+ bool RemovedAllFiles = true;
+ for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
+ File != FileEnd && !EC; File.increment(EC)) {
+ // We only care about module and global module index files.
+ if (llvm::sys::path::extension(File->path()) != ".pcm" &&
+ llvm::sys::path::filename(File->path()) != "modules.idx") {
+ RemovedAllFiles = false;
+ continue;
+ }
+
+ // Look at this file. If we can't stat it, there's nothing interesting
+ // there.
+ if (::stat(File->path().c_str(), &StatBuf)) {
+ RemovedAllFiles = false;
+ continue;
+ }
+
+ // If the file has been used recently enough, leave it there.
+ time_t FileAccessTime = StatBuf.st_atime;
+ if (CurrentTime - FileAccessTime <= HSOpts.ModuleCachePruneAfter) {
+ RemovedAllFiles = false;;
+ continue;
+ }
+
+ // Remove the file.
+ bool Existed;
+ if (llvm::sys::fs::remove(File->path(), Existed) || !Existed) {
+ RemovedAllFiles = false;
+ }
+ }
+
+ // If we removed all of the files in the directory, remove the directory
+ // itself.
+ if (RemovedAllFiles) {
+ bool Existed;
+ llvm::sys::fs::remove(Dir->path(), Existed);
+ }
+ }
+}
+
ModuleLoadResult
CompilerInstance::loadModule(SourceLocation ImportLoc,
ModuleIdPath Path,
@@ -1042,6 +1135,14 @@ CompilerInstance::loadModule(SourceLocat
if (!hasASTContext())
createASTContext();
+ // If we're not recursively building a module, check whether we
+ // need to prune the module cache.
+ if (getSourceManager().getModuleBuildStack().empty() &&
+ getHeaderSearchOpts().ModuleCachePruneInterval > 0 &&
+ getHeaderSearchOpts().ModuleCachePruneAfter > 0) {
+ pruneModuleCache(getHeaderSearchOpts());
+ }
+
std::string Sysroot = getHeaderSearchOpts().Sysroot;
const PreprocessorOptions &PPOpts = getPreprocessorOpts();
ModuleManager = new ASTReader(getPreprocessor(), *Context,
Modified: cfe/trunk/lib/Frontend/CompilerInvocation.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInvocation.cpp?rev=177918&r1=177917&r2=177918&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/CompilerInvocation.cpp (original)
+++ cfe/trunk/lib/Frontend/CompilerInvocation.cpp Mon Mar 25 16:19:16 2013
@@ -836,7 +836,10 @@ static void ParseHeaderSearchArgs(Header
Opts.ResourceDir = Args.getLastArgValue(OPT_resource_dir);
Opts.ModuleCachePath = Args.getLastArgValue(OPT_fmodules_cache_path);
Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
-
+ Opts.ModuleCachePruneInterval
+ = Args.getLastArgIntValue(OPT_fmodules_prune_interval, 7*24*60*60);
+ Opts.ModuleCachePruneAfter
+ = Args.getLastArgIntValue(OPT_fmodules_prune_after, 31*24*60*60);
for (arg_iterator it = Args.filtered_begin(OPT_fmodules_ignore_macro),
ie = Args.filtered_end(); it != ie; ++it) {
StringRef MacroDef = (*it)->getValue();
Added: cfe/trunk/test/Modules/prune.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/prune.m?rev=177918&view=auto
==============================================================================
--- cfe/trunk/test/Modules/prune.m (added)
+++ cfe/trunk/test/Modules/prune.m Mon Mar 25 16:19:16 2013
@@ -0,0 +1,46 @@
+// Test the automatic pruning of module cache entries.
+#ifdef IMPORT_DEPENDS_ON_MODULE
+ at import DependsOnModule;
+#else
+ at import Module;
+#endif
+
+// We need 'touch' and 'find' for this test to work.
+// REQUIRES: shell
+
+// Clear out the module cache
+// RUN: rm -rf %t
+// Run Clang twice so we end up creating the timestamp file (the second time).
+// RUN: %clang_cc1 -DIMPORT_DEPENDS_ON_MODULE -fmodules-ignore-macro=DIMPORT_DEPENDS_ON_MODULE -fmodules -F %S/Inputs -fmodules-cache-path=%t %s -verify
+// RUN: %clang_cc1 -DIMPORT_DEPENDS_ON_MODULE -fmodules-ignore-macro=DIMPORT_DEPENDS_ON_MODULE -fmodules -F %S/Inputs -fmodules-cache-path=%t %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set the timestamp back more than two days. We should try to prune,
+// but nothing gets pruned because the module files are new enough.
+// RUN: touch -m -a -A -481200 %t/modules.timestamp
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set the DependsOnModule access time back more than four days.
+// This shouldn't prune anything, because the timestamp has been updated, so
+// the pruning mechanism won't fire.
+// RUN: touch -a -A -961200 `find /Volumes/Data/dgregor/Projects/llvm-build-xcode/tools/clang/test/Modules/Output/prune.m.tmp -name DependsOnModule.pcm`
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set both timestamp and DependsOnModule.pcm back beyond the cutoff.
+// This should trigger pruning, which will remove DependsOnModule but not Module.
+// RUN: touch -m -a -A -481200 %t/modules.timestamp
+// RUN: touch -a -A -961200 `find /Volumes/Data/dgregor/Projects/llvm-build-xcode/tools/clang/test/Modules/Output/prune.m.tmp -name DependsOnModule.pcm`
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | not grep DependsOnModule.pcm
+
+// expected-no-diagnostics
More information about the cfe-commits
mailing list