[llvm] r306126 - Add a ThinLTO cache policy for controlling the maximum cache size in bytes.

Peter Collingbourne via llvm-commits llvm-commits at lists.llvm.org
Fri Jun 23 10:05:04 PDT 2017


Author: pcc
Date: Fri Jun 23 12:05:03 2017
New Revision: 306126

URL: http://llvm.org/viewvc/llvm-project?rev=306126&view=rev
Log:
Add a ThinLTO cache policy for controlling the maximum cache size in bytes.

This is useful when an upper limit on the cache size needs to be
controlled independently of the amount of the amount of free space.

One use case is a machine with a large number of cache directories
(e.g. a buildbot slave hosting a large number of independent build
jobs). By imposing an upper size limit on each cache directory,
users can more easily estimate the server's capacity.

Differential Revision: https://reviews.llvm.org/D34547

Modified:
    llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
    llvm/trunk/include/llvm/Support/CachePruning.h
    llvm/trunk/lib/Support/CachePruning.cpp
    llvm/trunk/unittests/Support/CachePruningTest.cpp

Modified: llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h?rev=306126&r1=306125&r2=306126&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h (original)
+++ llvm/trunk/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h Fri Jun 23 12:05:03 2017
@@ -177,7 +177,7 @@ public:
    */
   void setMaxCacheSizeRelativeToAvailableSpace(unsigned Percentage) {
     if (Percentage)
-      CacheOptions.Policy.PercentageOfAvailableSpace = Percentage;
+      CacheOptions.Policy.MaxSizePercentageOfAvailableSpace = Percentage;
   }
 
   /**@}*/

Modified: llvm/trunk/include/llvm/Support/CachePruning.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Support/CachePruning.h?rev=306126&r1=306125&r2=306126&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Support/CachePruning.h (original)
+++ llvm/trunk/include/llvm/Support/CachePruning.h Fri Jun 23 12:05:03 2017
@@ -39,8 +39,13 @@ struct CachePruningPolicy {
   /// available space on the the disk. Set to 100 to indicate no limit, 50 to
   /// indicate that the cache size will not be left over half the available disk
   /// space. A value over 100 will be reduced to 100. A value of 0 disables the
-  /// size-based pruning.
-  unsigned PercentageOfAvailableSpace = 75;
+  /// percentage size-based pruning.
+  unsigned MaxSizePercentageOfAvailableSpace = 75;
+
+  /// The maximum size for the cache directory in bytes. A value over the amount
+  /// of available space on the disk will be reduced to the amount of available
+  /// space. A value of 0 disables the absolute size-based pruning.
+  uint64_t MaxSizeBytes = 0;
 };
 
 /// Parse the given string as a cache pruning policy. Defaults are taken from a

Modified: llvm/trunk/lib/Support/CachePruning.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Support/CachePruning.cpp?rev=306126&r1=306125&r2=306126&view=diff
==============================================================================
--- llvm/trunk/lib/Support/CachePruning.cpp (original)
+++ llvm/trunk/lib/Support/CachePruning.cpp Fri Jun 23 12:05:03 2017
@@ -79,10 +79,10 @@ llvm::parseCachePruningPolicy(StringRef
         return DurationOrErr.takeError();
       Policy.Expiration = *DurationOrErr;
     } else if (Key == "cache_size") {
-      if (Value.back() != '%')
-        return make_error<StringError>("'" + Value + "' must be a percentage",
-                                       inconvertibleErrorCode());
-      StringRef SizeStr = Value.slice(0, Value.size() - 1);
+    if (Value.back() != '%')
+      return make_error<StringError>("'" + Value + "' must be a percentage",
+                                     inconvertibleErrorCode());
+      StringRef SizeStr = Value.drop_back();
       uint64_t Size;
       if (SizeStr.getAsInteger(0, Size))
         return make_error<StringError>("'" + SizeStr + "' not an integer",
@@ -91,7 +91,28 @@ llvm::parseCachePruningPolicy(StringRef
         return make_error<StringError>("'" + SizeStr +
                                            "' must be between 0 and 100",
                                        inconvertibleErrorCode());
-      Policy.PercentageOfAvailableSpace = Size;
+      Policy.MaxSizePercentageOfAvailableSpace = Size;
+    } else if (Key == "cache_size_bytes") {
+      uint64_t Mult = 1;
+      switch (Value.back()) {
+      case 'k':
+        Mult = 1024;
+        Value = Value.drop_back();
+        break;
+      case 'm':
+        Mult = 1024 * 1024;
+        Value = Value.drop_back();
+        break;
+      case 'g':
+        Mult = 1024 * 1024 * 1024;
+        Value = Value.drop_back();
+        break;
+      }
+      uint64_t Size;
+      if (Value.getAsInteger(0, Size))
+        return make_error<StringError>("'" + Value + "' not an integer",
+                                       inconvertibleErrorCode());
+      Policy.MaxSizeBytes = Size * Mult;
     } else {
       return make_error<StringError>("Unknown key: '" + Key + "'",
                                      inconvertibleErrorCode());
@@ -115,11 +136,12 @@ bool llvm::pruneCache(StringRef Path, Ca
   if (!isPathDir)
     return false;
 
-  Policy.PercentageOfAvailableSpace =
-      std::min(Policy.PercentageOfAvailableSpace, 100u);
+  Policy.MaxSizePercentageOfAvailableSpace =
+      std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
 
   if (Policy.Expiration == seconds(0) &&
-      Policy.PercentageOfAvailableSpace == 0) {
+      Policy.MaxSizePercentageOfAvailableSpace == 0 &&
+      Policy.MaxSizeBytes == 0) {
     DEBUG(dbgs() << "No pruning settings set, exit early\n");
     // Nothing will be pruned, early exit
     return false;
@@ -157,7 +179,8 @@ bool llvm::pruneCache(StringRef Path, Ca
     writeTimestampFile(TimestampFile);
   }
 
-  bool ShouldComputeSize = (Policy.PercentageOfAvailableSpace > 0);
+  bool ShouldComputeSize =
+      (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
 
   // Keep track of space
   std::set<std::pair<uint64_t, std::string>> FileSizes;
@@ -216,14 +239,22 @@ bool llvm::pruneCache(StringRef Path, Ca
     }
     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     auto AvailableSpace = TotalSize + SpaceInfo.free;
-    auto FileAndSize = FileSizes.rbegin();
+
+    if (Policy.MaxSizePercentageOfAvailableSpace == 0)
+      Policy.MaxSizePercentageOfAvailableSpace = 100;
+    if (Policy.MaxSizeBytes == 0)
+      Policy.MaxSizeBytes = AvailableSpace;
+    auto TotalSizeTarget = std::min<uint64_t>(
+        AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
+        Policy.MaxSizeBytes);
+
     DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
-                 << "% target is: " << Policy.PercentageOfAvailableSpace
-                 << "\n");
+                 << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
+                 << "%, " << Policy.MaxSizeBytes << " bytes\n");
+
+    auto FileAndSize = FileSizes.rbegin();
     // Remove the oldest accessed files first, till we get below the threshold
-    while (((100 * TotalSize) / AvailableSpace) >
-               Policy.PercentageOfAvailableSpace &&
-           FileAndSize != FileSizes.rend()) {
+    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
       // Remove the file.
       sys::fs::remove(FileAndSize->second);
       // Update size

Modified: llvm/trunk/unittests/Support/CachePruningTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Support/CachePruningTest.cpp?rev=306126&r1=306125&r2=306126&view=diff
==============================================================================
--- llvm/trunk/unittests/Support/CachePruningTest.cpp (original)
+++ llvm/trunk/unittests/Support/CachePruningTest.cpp Fri Jun 23 12:05:03 2017
@@ -18,7 +18,7 @@ TEST(CachePruningPolicyParser, Empty) {
   ASSERT_TRUE(bool(P));
   EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
   EXPECT_EQ(std::chrono::hours(7 * 24), P->Expiration);
-  EXPECT_EQ(75u, P->PercentageOfAvailableSpace);
+  EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
 }
 
 TEST(CachePruningPolicyParser, Interval) {
@@ -39,10 +39,30 @@ TEST(CachePruningPolicyParser, Expiratio
   EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
 }
 
-TEST(CachePruningPolicyParser, PercentageOfAvailableSpace) {
+TEST(CachePruningPolicyParser, MaxSizePercentageOfAvailableSpace) {
   auto P = parseCachePruningPolicy("cache_size=100%");
   ASSERT_TRUE(bool(P));
-  EXPECT_EQ(100u, P->PercentageOfAvailableSpace);
+  EXPECT_EQ(100u, P->MaxSizePercentageOfAvailableSpace);
+  EXPECT_EQ(0u, P->MaxSizeBytes);
+}
+
+TEST(CachePruningPolicyParser, MaxSizeBytes) {
+  auto P = parseCachePruningPolicy("cache_size_bytes=1");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
+  EXPECT_EQ(1u, P->MaxSizeBytes);
+  P = parseCachePruningPolicy("cache_size_bytes=2k");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
+  EXPECT_EQ(2u * 1024u, P->MaxSizeBytes);
+  P = parseCachePruningPolicy("cache_size_bytes=3m");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
+  EXPECT_EQ(3u * 1024u * 1024u, P->MaxSizeBytes);
+  P = parseCachePruningPolicy("cache_size_bytes=4g");
+  ASSERT_TRUE(bool(P));
+  EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
+  EXPECT_EQ(4ull * 1024ull * 1024ull * 1024ull, P->MaxSizeBytes);
 }
 
 TEST(CachePruningPolicyParser, Multiple) {
@@ -50,7 +70,7 @@ TEST(CachePruningPolicyParser, Multiple)
   ASSERT_TRUE(bool(P));
   EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
   EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
-  EXPECT_EQ(50u, P->PercentageOfAvailableSpace);
+  EXPECT_EQ(50u, P->MaxSizePercentageOfAvailableSpace);
 }
 
 TEST(CachePruningPolicyParser, Errors) {
@@ -66,6 +86,12 @@ TEST(CachePruningPolicyParser, Errors) {
             toString(parseCachePruningPolicy("cache_size=foo%").takeError()));
   EXPECT_EQ("'101' must be between 0 and 100",
             toString(parseCachePruningPolicy("cache_size=101%").takeError()));
+  EXPECT_EQ(
+      "'foo' not an integer",
+      toString(parseCachePruningPolicy("cache_size_bytes=foo").takeError()));
+  EXPECT_EQ(
+      "'foo' not an integer",
+      toString(parseCachePruningPolicy("cache_size_bytes=foom").takeError()));
   EXPECT_EQ("Unknown key: 'foo'",
             toString(parseCachePruningPolicy("foo=bar").takeError()));
 }




More information about the llvm-commits mailing list