[llvm] r340374 - [Support][CachePruning] prune least recently accessed files first
Bob Haarman via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 21 17:52:16 PDT 2018
Author: inglorion
Date: Tue Aug 21 17:52:16 2018
New Revision: 340374
URL: http://llvm.org/viewvc/llvm-project?rev=340374&view=rev
Log:
[Support][CachePruning] prune least recently accessed files first
Summary:
Before this change, pruning order was based on size. This changes it
to be based on time of last use instead, preferring to keep recently
used files and prune older ones.
Reviewers: pcc, rnk, espindola
Reviewed By: rnk
Subscribers: emaste, arichardson, hiraditya, steven_wu, dexonsmith, llvm-commits
Differential Revision: https://reviews.llvm.org/D51062
Modified:
llvm/trunk/lib/Support/CachePruning.cpp
llvm/trunk/test/ThinLTO/X86/cache.ll
Modified: llvm/trunk/lib/Support/CachePruning.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CachePruning.cpp?rev=340374&r1=340373&r2=340374&view=diff
==============================================================================
--- llvm/trunk/lib/Support/CachePruning.cpp (original)
+++ llvm/trunk/lib/Support/CachePruning.cpp Tue Aug 21 17:52:16 2018
@@ -27,6 +27,28 @@
using namespace llvm;
+namespace {
+struct FileInfo {
+ sys::TimePoint<> Time;
+ uint64_t Size;
+ std::string Path;
+
+ /// Used to determine which files to prune first. Also used to determine
+ /// set membership, so must take into account all fields.
+ bool operator<(const FileInfo &Other) const {
+ if (Time < Other.Time)
+ return true;
+ else if (Other.Time < Time)
+ return false;
+ if (Other.Size < Size)
+ return true;
+ else if (Size < Other.Size)
+ return false;
+ return Path < Other.Path;
+ }
+};
+} // anonymous namespace
+
/// Write a new timestamp file with the given path. This is used for the pruning
/// interval option.
static void writeTimestampFile(StringRef TimestampFile) {
@@ -185,8 +207,9 @@ bool llvm::pruneCache(StringRef Path, Ca
writeTimestampFile(TimestampFile);
}
- // Keep track of space. Needs to be kept ordered by size for determinism.
- std::set<std::pair<uint64_t, std::string>> FileSizes;
+ // Keep track of files to delete to get below the size limit.
+ // Order by time of last use so that recently used files are preserved.
+ std::set<FileInfo> FileInfos;
uint64_t TotalSize = 0;
// Walk the entire directory cache, looking for unused files.
@@ -224,22 +247,22 @@ bool llvm::pruneCache(StringRef Path, Ca
// Leave it here for now, but add it to the list of size-based pruning.
TotalSize += StatusOrErr->getSize();
- FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
+ FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
}
- auto FileAndSize = FileSizes.rbegin();
- size_t NumFiles = FileSizes.size();
+ auto FileInfo = FileInfos.begin();
+ size_t NumFiles = FileInfos.size();
auto RemoveCacheFile = [&]() {
// Remove the file.
- sys::fs::remove(FileAndSize->second);
+ sys::fs::remove(FileInfo->Path);
// Update size
- TotalSize -= FileAndSize->first;
+ TotalSize -= FileInfo->Size;
NumFiles--;
- LLVM_DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
- << FileAndSize->first << "), new occupancy is "
- << TotalSize << "%\n");
- ++FileAndSize;
+ LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
+ << FileInfo->Size << "), new occupancy is " << TotalSize
+ << "%\n");
+ ++FileInfo;
};
// Prune for number of files.
@@ -270,7 +293,7 @@ bool llvm::pruneCache(StringRef Path, Ca
<< Policy.MaxSizeBytes << " bytes\n");
// Remove the oldest accessed files first, till we get below the threshold.
- while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
+ while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
RemoveCacheFile();
}
return true;
Modified: llvm/trunk/test/ThinLTO/X86/cache.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/ThinLTO/X86/cache.ll?rev=340374&r1=340373&r2=340374&view=diff
==============================================================================
--- llvm/trunk/test/ThinLTO/X86/cache.ll (original)
+++ llvm/trunk/test/ThinLTO/X86/cache.ll Tue Aug 21 17:52:16 2018
@@ -102,38 +102,44 @@
; RUN: ls -ltu %t.cache/* | not grep 1970-01-01
; Verify that specifying max size for the cache directory prunes it to this
-; size, removing the largest files first.
+; size, removing the oldest files first.
; RUN: rm -Rf %t.cache && mkdir %t.cache
; Create cache files with different sizes.
-; Only 8B, 16B and 76B files should stay after pruning.
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-1024', 'w') as file: file.truncate(1024)"
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-16', 'w') as file: file.truncate(16)"
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-8', 'w') as file: file.truncate(8)"
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-76', 'w') as file: file.truncate(76)"
-; RUN: %python -c "with open(r'%t.cache/llvmcache-foo-77', 'w') as file: file.truncate(77)"
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-bytes 100
-; RUN: ls %t.cache/llvmcache-foo-16
+; Only 8B and 76B files should stay after pruning.
+; RUN: "%python" -c "with open(r'%t.cache/llvmcache-foo-100k', 'w') as file: file.truncate(102400)"
+; RUN: touch -t 198002011200 %t.cache/llvmcache-foo-100k
+; RUN: "%python" -c "with open(r'%t.cache/llvmcache-foo-16', 'w') as file: file.truncate(16)"
+; RUN: touch -t 198002021200 %t.cache/llvmcache-foo-16
+; RUN: "%python" -c "with open(r'%t.cache/llvmcache-foo-77k', 'w') as file: file.truncate(78848)"
+; RUN: touch -t 198002031200 %t.cache/llvmcache-foo-77k
+; RUN: "%python" -c "with open(r'%t.cache/llvmcache-foo-8', 'w') as file: file.truncate(8)"
+; RUN: "%python" -c "with open(r'%t.cache/llvmcache-foo-76', 'w') as file: file.truncate(76)"
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-bytes 78847
; RUN: ls %t.cache/llvmcache-foo-8
; RUN: ls %t.cache/llvmcache-foo-76
-; RUN: not ls %t.cache/llvmcache-foo-1024
-; RUN: not ls %t.cache/llvmcache-foo-77
+; RUN: not ls %t.cache/llvmcache-foo-16
+; RUN: not ls %t.cache/llvmcache-foo-100k
+; RUN: not ls %t.cache/llvmcache-foo-77k
; Verify that specifying max number of files in the cache directory prunes
-; it to this amount, removing the largest files first.
+; it to this amount, removing the oldest files first.
; RUN: rm -Rf %t.cache && mkdir %t.cache
; Create cache files with different sizes.
-; Only 8B and 16B files should stay after pruning.
-; RUN: %python -c "print(' ' * 1023)" > %t.cache/llvmcache-foo-1024
-; RUN: %python -c "print(' ' * 15)" > %t.cache/llvmcache-foo-16
-; RUN: %python -c "print(' ' * 7)" > %t.cache/llvmcache-foo-8
-; RUN: %python -c "print(' ' * 75)" > %t.cache/llvmcache-foo-76
-; RUN: %python -c "print(' ' * 76)" > %t.cache/llvmcache-foo-77
-; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-files 2
-; RUN: ls %t.cache/llvmcache-foo-16
-; RUN: ls %t.cache/llvmcache-foo-8
-; RUN: not ls %t.cache/llvmcache-foo-76
+; Only 75B and 76B files should stay after pruning.
+; RUN: "%python" -c "print(' ' * 1023)" > %t.cache/llvmcache-foo-1023
+; RUN: touch -t 198002011200 %t.cache/llvmcache-foo-1023
+; RUN: "%python" -c "print(' ' * 15)" > %t.cache/llvmcache-foo-15
+; RUN: touch -t 198002021200 %t.cache/llvmcache-foo-15
+; RUN: "%python" -c "print(' ' * 7)" > %t.cache/llvmcache-foo-7
+; RUN: touch -t 198002031200 %t.cache/llvmcache-foo-7
+; RUN: "%python" -c "print(' ' * 75)" > %t.cache/llvmcache-foo-75
+; RUN: "%python" -c "print(' ' * 76)" > %t.cache/llvmcache-foo-76
+; RUN: llvm-lto -thinlto-action=run -exported-symbol=globalfunc %t2.bc %t.bc -thinlto-cache-dir %t.cache --thinlto-cache-max-size-files 4
+; RUN: ls %t.cache/llvmcache-foo-75
+; RUN: ls %t.cache/llvmcache-foo-76
+; RUN: not ls %t.cache/llvmcache-foo-15
; RUN: not ls %t.cache/llvmcache-foo-1024
-; RUN: not ls %t.cache/llvmcache-foo-77
+; RUN: not ls %t.cache/llvmcache-foo-7
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.11.0"
More information about the llvm-commits
mailing list