[llvm] r265209 - Add Cache Pruning support
Mehdi Amini via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 1 20:28:27 PDT 2016
Author: mehdi_amini
Date: Fri Apr 1 22:28:26 2016
New Revision: 265209
URL: http://llvm.org/viewvc/llvm-project?rev=265209&view=rev
Log:
Add Cache Pruning support
Incremental LTO will usea cache to store object files.
This patch handles the pruning part of the cache, exposing
a few knobs:
- Pruning interval: the implementation keeps a "timestamp" file in the
directory and will scan it only after a given interval since the
last modification of the timestamp file. This is for performance
purpose, we don't want to scan continuously the folder.
- Entry expiration: this is the time after which a file that hasn't
been used is remove from the cache.
- Maximum size: expressed in percentage of the available disk space,
it helps to avoid that we blow up the disk space.
http://reviews.llvm.org/D18422
From: Mehdi Amini <mehdi.amini at apple.com>
Added:
llvm/trunk/include/llvm/Support/CachePruning.h
llvm/trunk/lib/Support/CachePruning.cpp
Modified:
llvm/trunk/lib/Support/CMakeLists.txt
Added: llvm/trunk/include/llvm/Support/CachePruning.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/CachePruning.h?rev=265209&view=auto
==============================================================================
--- llvm/trunk/include/llvm/Support/CachePruning.h (added)
+++ llvm/trunk/include/llvm/Support/CachePruning.h Fri Apr 1 22:28:26 2016
@@ -0,0 +1,69 @@
+//=- CachePruning.h - Helper to manage the pruning of a cache dir -*- C++ -*-=//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pruning of a directory intended for cache storage, using
+// various policies.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_CACHE_PRUNING_H
+#define LLVM_SUPPORT_CACHE_PRUNING_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+
+/// Handle pruning a directory provided a path and some options to control what
+/// to prune.
+class CachePruning {
+public:
+ /// Prepare to prune \p Path.
+ CachePruning(StringRef Path) : Path(Path) {}
+
+ /// Define the pruning interval. This is intended to be used to avoid scanning
+ /// the directory too often. It does not impact the decision of which file to
+ /// prune. A value of 0 forces the scan to occurs.
+ CachePruning &setPruningInterval(int PruningInterval) {
+ Interval = PruningInterval;
+ return *this;
+ }
+
+ /// Define the expiration for a file. When a file hasn't been accessed for
+ /// \p ExpireAfter seconds, it is removed from the cache. A value of 0 disable
+ /// the expiration-based pruning.
+ CachePruning &setEntryExpiration(unsigned ExpireAfter) {
+ Expiration = ExpireAfter;
+ return *this;
+ }
+
+ /// Define the maximum size for the cache directory, in terms of percentage of
+ /// the available space on the the disk. Set to 100 to indicate no limit, 50
+ /// to indicate that the cache size will not be left over half the
+ /// available disk space. A value over 100 will be reduced to 100. A value of
+ /// 0 disable the size-based pruning.
+ CachePruning &setMaxSize(unsigned Percentage) {
+ PercentageOfAvailableSpace = std::min(100u, Percentage);
+ return *this;
+ }
+
+ /// Peform pruning using the supplied options, returns true if pruning
+ /// occured, i.e. if PruningInterval was expired.
+ bool prune();
+
+private:
+ // Options that matches the setters above.
+ std::string Path;
+ unsigned Expiration = 0;
+ unsigned Interval = 0;
+ unsigned PercentageOfAvailableSpace = 0;
+};
+
+} // namespace llvm
+
+#endif
\ No newline at end of file
Modified: llvm/trunk/lib/Support/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CMakeLists.txt?rev=265209&r1=265208&r2=265209&view=diff
==============================================================================
--- llvm/trunk/lib/Support/CMakeLists.txt (original)
+++ llvm/trunk/lib/Support/CMakeLists.txt Fri Apr 1 22:28:26 2016
@@ -35,6 +35,7 @@ add_llvm_library(LLVMSupport
Allocator.cpp
BlockFrequency.cpp
BranchProbability.cpp
+ CachePruning.cpp
circular_raw_ostream.cpp
COM.cpp
CommandLine.cpp
Added: llvm/trunk/lib/Support/CachePruning.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CachePruning.cpp?rev=265209&view=auto
==============================================================================
--- llvm/trunk/lib/Support/CachePruning.cpp (added)
+++ llvm/trunk/lib/Support/CachePruning.cpp Fri Apr 1 22:28:26 2016
@@ -0,0 +1,130 @@
+//===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the pruning of a directory based on least recently used.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CachePruning.h"
+
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <set>
+
+using namespace llvm;
+
+/// Write a new timestamp file with the given path. This is used for the pruning
+/// interval option.
+static void writeTimestampFile(StringRef TimestampFile) {
+ std::error_code EC;
+ raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
+}
+
+/// Prune the cache of files that haven't been accessed in a long time.
+bool CachePruning::prune() {
+ SmallString<128> TimestampFile(Path);
+ sys::path::append(TimestampFile, "llvmcache.timestamp");
+
+ if (Expiration == 0 && PercentageOfAvailableSpace == 0)
+ // Nothing will be pruned, early exit
+ return false;
+
+ // Try to stat() the timestamp file.
+ sys::fs::file_status FileStatus;
+ sys::TimeValue CurrentTime = sys::TimeValue::now();
+ if (sys::fs::status(TimestampFile, FileStatus)) {
+ if (errno == ENOENT) {
+ // If the timestamp file wasn't there, create one now.
+ writeTimestampFile(TimestampFile);
+ } else {
+ // Unknown error?
+ return false;
+ }
+ } else {
+ if (Interval) {
+ // Check whether the time stamp is older than our pruning interval.
+ // If not, do nothing.
+ sys::TimeValue TimeStampModTime = FileStatus.getLastModificationTime();
+ auto TimeInterval = sys::TimeValue(sys::TimeValue::SecondsType(Interval));
+ if (CurrentTime - TimeStampModTime <= TimeInterval)
+ return false;
+ }
+ // Write a new timestamp file so that nobody else attempts to prune.
+ // There is a benign race condition here, if two processes happen to
+ // notice at the same time that the timestamp is out-of-date.
+ writeTimestampFile(TimestampFile);
+ }
+
+ bool ShouldComputeSize = (PercentageOfAvailableSpace > 0);
+
+ // Keep track of space
+ std::set<std::pair<uint64_t, std::string>> FileSizes;
+ uint64_t TotalSize = 0;
+ // Helper to add a path to the set of files to consider for size-based
+ // pruning, sorted by last accessed time.
+ auto AddToFileListForSizePruning =
+ [&](StringRef Path, sys::TimeValue FileAccessTime) {
+ if (!ShouldComputeSize)
+ return;
+ TotalSize += FileStatus.getSize();
+ FileSizes.insert(
+ std::make_pair(FileAccessTime.seconds(), std::string(Path)));
+ };
+
+ // Walk the entire directory cache, looking for unused files.
+ std::error_code EC;
+ SmallString<128> CachePathNative;
+ sys::path::native(Path, CachePathNative);
+ auto TimeExpiration = sys::TimeValue(sys::TimeValue::SecondsType(Expiration));
+ // Walk all of the files within this directory.
+ for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
+ File != FileEnd && !EC; File.increment(EC)) {
+ // Do not touch the timestamp.
+ if (File->path() == TimestampFile)
+ continue;
+
+ // Look at this file. If we can't stat it, there's nothing interesting
+ // there.
+ if (sys::fs::status(File->path(), FileStatus))
+ continue;
+
+ // If the file hasn't been used recently enough, delete it
+ sys::TimeValue FileAccessTime = FileStatus.getLastAccessedTime();
+ if (CurrentTime - FileAccessTime > TimeExpiration) {
+ sys::fs::remove(File->path());
+ continue;
+ }
+
+ // Leave it here for now, but add it to the list of size-based pruning.
+ AddToFileListForSizePruning(File->path(), FileAccessTime);
+ }
+
+ // Prune for size now if needed
+ if (ShouldComputeSize) {
+ auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
+ if (!ErrOrSpaceInfo) {
+ report_fatal_error("Can't get available size");
+ }
+ sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
+ auto AvailableSpace = TotalSize + SpaceInfo.free;
+ auto FileAndSize = FileSizes.rbegin();
+ // Remove the oldest accessed files first, till we get below the threshold
+ while (((100 * TotalSize) / AvailableSpace) > PercentageOfAvailableSpace &&
+ FileAndSize != FileSizes.rend()) {
+ // Remove the file.
+ sys::fs::remove(FileAndSize->second);
+ // Update size
+ TotalSize -= FileAndSize->first;
+ ++FileAndSize;
+ }
+ }
+ return true;
+}
More information about the llvm-commits
mailing list