[llvm] r318857 - CachePruning: Allow limiting the number of files in the cache directory.
Peter Collingbourne via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 22 10:27:31 PST 2017
Author: pcc
Date: Wed Nov 22 10:27:31 2017
New Revision: 318857
URL: http://llvm.org/viewvc/llvm-project?rev=318857&view=rev
Log:
CachePruning: Allow limiting the number of files in the cache directory.
The default limit is 1000000 but it can be configured with a cache
policy. The motivation is that some filesystems (notably ext4) have
a limit on the number of files that can be contained in a directory
(separate from the inode limit).
Differential Revision: https://reviews.llvm.org/D40327
Modified:
llvm/trunk/include/llvm/Support/CachePruning.h
llvm/trunk/lib/Support/CachePruning.cpp
Modified: llvm/trunk/include/llvm/Support/CachePruning.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/CachePruning.h?rev=318857&r1=318856&r2=318857&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Support/CachePruning.h (original)
+++ llvm/trunk/include/llvm/Support/CachePruning.h Wed Nov 22 10:27:31 2017
@@ -46,6 +46,15 @@ struct CachePruningPolicy {
/// of available space on the disk will be reduced to the amount of available
/// space. A value of 0 disables the absolute size-based pruning.
uint64_t MaxSizeBytes = 0;
+
+ /// The maximum number of files in the cache directory. A value of 0 disables
+ /// the number of files based pruning.
+ ///
+ /// This defaults to 1000000 because with that many files there are
+ /// diminishing returns on the effectiveness of the cache, and some file
+ /// systems have a limit on how many files can be contained in a directory
+ /// (notably ext4, which is limited to around 6000000 files).
+ uint64_t MaxSizeFiles = 1000000;
};
/// Parse the given string as a cache pruning policy. Defaults are taken from a
Modified: llvm/trunk/lib/Support/CachePruning.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CachePruning.cpp?rev=318857&r1=318856&r2=318857&view=diff
==============================================================================
--- llvm/trunk/lib/Support/CachePruning.cpp (original)
+++ llvm/trunk/lib/Support/CachePruning.cpp Wed Nov 22 10:27:31 2017
@@ -113,6 +113,10 @@ llvm::parseCachePruningPolicy(StringRef
return make_error<StringError>("'" + Value + "' not an integer",
inconvertibleErrorCode());
Policy.MaxSizeBytes = Size * Mult;
+ } else if (Key == "cache_size_files") {
+ if (Value.getAsInteger(0, Policy.MaxSizeFiles))
+ return make_error<StringError>("'" + Value + "' not an integer",
+ inconvertibleErrorCode());
} else {
return make_error<StringError>("Unknown key: '" + Key + "'",
inconvertibleErrorCode());
@@ -141,7 +145,7 @@ bool llvm::pruneCache(StringRef Path, Ca
if (Policy.Expiration == seconds(0) &&
Policy.MaxSizePercentageOfAvailableSpace == 0 &&
- Policy.MaxSizeBytes == 0) {
+ Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
DEBUG(dbgs() << "No pruning settings set, exit early\n");
// Nothing will be pruned, early exit
return false;
@@ -179,9 +183,6 @@ bool llvm::pruneCache(StringRef Path, Ca
writeTimestampFile(TimestampFile);
}
- bool ShouldComputeSize =
- (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
-
// Keep track of space. Needs to be kept ordered by size for determinism.
std::set<std::pair<uint64_t, std::string>> FileSizes;
uint64_t TotalSize = 0;
@@ -211,7 +212,7 @@ bool llvm::pruneCache(StringRef Path, Ca
// If the file hasn't been used recently enough, delete it
const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
auto FileAge = CurrentTime - FileAccessTime;
- if (FileAge > Policy.Expiration) {
+ if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
DEBUG(dbgs() << "Remove " << File->path() << " ("
<< duration_cast<seconds>(FileAge).count() << "s old)\n");
sys::fs::remove(File->path());
@@ -219,14 +220,32 @@ bool llvm::pruneCache(StringRef Path, Ca
}
// Leave it here for now, but add it to the list of size-based pruning.
- if (!ShouldComputeSize)
- continue;
TotalSize += StatusOrErr->getSize();
FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
}
+ auto FileAndSize = FileSizes.rbegin();
+ size_t NumFiles = FileSizes.size();
+
+ auto RemoveCacheFile = [&]() {
+ // Remove the file.
+ sys::fs::remove(FileAndSize->second);
+ // Update size
+ TotalSize -= FileAndSize->first;
+ NumFiles--;
+ DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
+ << FileAndSize->first << "), new occupancy is " << TotalSize
+ << "%\n");
+ ++FileAndSize;
+ };
+
+ // Prune for number of files.
+ if (Policy.MaxSizeFiles)
+ while (NumFiles > Policy.MaxSizeFiles)
+ RemoveCacheFile();
+
// Prune for size now if needed
- if (ShouldComputeSize) {
+ if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
if (!ErrOrSpaceInfo) {
report_fatal_error("Can't get available size");
@@ -246,18 +265,9 @@ bool llvm::pruneCache(StringRef Path, Ca
<< "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
<< "%, " << Policy.MaxSizeBytes << " bytes\n");
- auto FileAndSize = FileSizes.rbegin();
- // Remove the oldest accessed files first, till we get below the threshold
- while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
- // Remove the file.
- sys::fs::remove(FileAndSize->second);
- // Update size
- TotalSize -= FileAndSize->first;
- DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
- << FileAndSize->first << "), new occupancy is " << TotalSize
- << "%\n");
- ++FileAndSize;
- }
+ // Remove the oldest accessed files first, till we get below the threshold.
+ while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
+ RemoveCacheFile();
}
return true;
}
More information about the llvm-commits
mailing list