[llvm] [CAS] Add UnifiedOnDiskCache and OnDiskCAS (PR #114103)
Justin Bogner via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 28 10:02:05 PDT 2025
================
@@ -0,0 +1,612 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Encapsulates \p OnDiskGraphDB and \p OnDiskKeyValueDB instances within one
+/// directory while also restricting storage growth with a scheme of chaining
+/// the two most recent directories (primary & upstream), where the primary
+/// "faults-in" data from the upstream one. When the primary (most recent)
+/// directory exceeds its intended limit a new empty directory becomes the
+/// primary one.
+///
+/// Within the top-level directory (the path that \p UnifiedOnDiskCache::open
+/// receives) there are directories named like this:
+///
+/// 'v<version>.<x>'
+/// 'v<version>.<x+1'
+/// 'v<version>.<x+2>'
+/// ...
+///
+/// 'version' is the version integer for this \p UnifiedOnDiskCache's scheme and
+/// the part after the dot is an increasing integer. The primary directory is
+/// the one with the highest integer and the upstream one is the directory
+/// before it. For example, if the sub-directories contained are:
+///
+/// 'v1.5', 'v1.6', 'v1.7', 'v1.8'
+///
+/// Then the primary one is 'v1.8', the upstream one is 'v1.7', and the rest are
+/// unused directories that can be safely deleted at any time and by any
+/// process.
+///
+/// Contained within the top-level directory is a file named "lock" which is
+/// used for processes to take shared or exclusive locks for the contents of the
+/// top directory. While a \p UnifiedOnDiskCache is open it keeps a shared lock
+/// for the top-level directory; when it closes, if the primary sub-directory
+/// exceeded its limit, it attempts to get an exclusive lock in order to create
+/// a new empty primary directory; if it can't get the exclusive lock it gives
+/// up and lets the next \p UnifiedOnDiskCache instance that closes to attempt
+/// again.
+///
+/// The downside of this scheme is that while \p UnifiedOnDiskCache is open on a
+/// directory, by any process, the storage size in that directory will keep
+/// growing unrestricted. But the major benefit is that garbage-collection can
+/// be triggered on a directory concurrently, at any time and by any process,
+/// without affecting any active readers/writers in the same process or other
+/// processes.
+///
+/// The \c UnifiedOnDiskCache also provides validation and recovery on top of
+/// the underlying on-disk storage. The low-level storage is designed to remain
+/// coherent across regular process crashes, but may be invalid after power loss
+/// or similar system failures. \c UnifiedOnDiskCache::validateIfNeeded allows
+/// validating the contents once per boot and can recover by marking invalid
+/// data for garbage collection.
+///
+/// The data recovery described above requires exclusive access to the CAS, and
+/// it is an error to attempt recovery if the CAS is open in any process/thread.
+/// In order to maximize backwards compatibility with tools that do not perform
+/// validation before opening the CAS, we do not attempt to get exclusive access
+/// until recovery is actually performed, meaning as long as the data is valid
+/// it will not conflict with concurrent use.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/UnifiedOnDiskCache.h"
+#include "BuiltinCAS.h"
+#include "OnDiskCommon.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/OnDiskGraphDB.h"
+#include "llvm/CAS/OnDiskKeyValueDB.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
+
+#if __has_include(<sys/sysctl.h>)
+#include <sys/sysctl.h>
+#endif
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::ondisk;
+
+/// FIXME: When the version of \p DBDirPrefix is bumped up we need to figure out
+/// how to handle the leftover sub-directories of the previous version, within
+/// the \p UnifiedOnDiskCache::collectGarbage function.
+static constexpr StringLiteral DBDirPrefix = "v1.";
+
+static constexpr StringLiteral ValidationFilename = "v1.validation";
+static constexpr StringLiteral CorruptPrefix = "corrupt.";
+
+ObjectID UnifiedOnDiskCache::getObjectIDFromValue(ArrayRef<char> Value) {
+ // little endian encoded.
+ assert(Value.size() == sizeof(uint64_t));
+ return ObjectID::fromOpaqueData(support::endian::read64le(Value.data()));
+}
+
+UnifiedOnDiskCache::ValueBytes
+UnifiedOnDiskCache::getValueFromObjectID(ObjectID ID) {
+ // little endian encoded.
+ UnifiedOnDiskCache::ValueBytes ValBytes;
+ static_assert(ValBytes.size() == sizeof(ID.getOpaqueData()));
+ support::endian::write64le(ValBytes.data(), ID.getOpaqueData());
+ return ValBytes;
+}
+
+Expected<std::optional<ArrayRef<char>>>
+UnifiedOnDiskCache::faultInFromUpstreamKV(ArrayRef<uint8_t> Key) {
+ assert(UpstreamGraphDB);
+ assert(UpstreamKVDB);
+
+ std::optional<ArrayRef<char>> UpstreamValue;
+ if (Error E = UpstreamKVDB->get(Key).moveInto(UpstreamValue))
+ return std::move(E);
+ if (!UpstreamValue)
+ return std::nullopt;
+
+ // The value is the \p ObjectID in the context of the upstream
+ // \p OnDiskGraphDB instance. Translate it to the context of the primary
+ // \p OnDiskGraphDB instance.
+ ObjectID UpstreamID = getObjectIDFromValue(*UpstreamValue);
+ auto PrimaryID =
+ PrimaryGraphDB->getReference(UpstreamGraphDB->getDigest(UpstreamID));
+ if (LLVM_UNLIKELY(!PrimaryID))
+ return PrimaryID.takeError();
+ return PrimaryKVDB->put(Key, getValueFromObjectID(*PrimaryID));
+}
+
+/// \returns all the 'v<version>.<x>' names of sub-directories, sorted with
+/// ascending order of the integer after the dot. Corrupt directories, if
+/// included, will come first.
+static Error getAllDBDirs(StringRef Path, SmallVectorImpl<std::string> &DBDirs,
+ bool IncludeCorrupt = false) {
+ struct DBDir {
+ uint64_t Order;
+ std::string Name;
+ };
+ SmallVector<DBDir, 6> FoundDBDirs;
+
+ std::error_code EC;
+ for (sys::fs::directory_iterator DirI(Path, EC), DirE; !EC && DirI != DirE;
+ DirI.increment(EC)) {
+ if (DirI->type() != sys::fs::file_type::directory_file)
+ continue;
+ StringRef SubDir = sys::path::filename(DirI->path());
+ if (IncludeCorrupt && SubDir.starts_with(CorruptPrefix)) {
+ FoundDBDirs.push_back({0, std::string(SubDir)});
+ continue;
+ }
+ if (!SubDir.starts_with(DBDirPrefix))
+ continue;
+ uint64_t Order;
+ if (SubDir.substr(DBDirPrefix.size()).getAsInteger(10, Order))
+ return createStringError(inconvertibleErrorCode(),
+ "unexpected directory " + DirI->path());
+ FoundDBDirs.push_back({Order, std::string(SubDir)});
+ }
+ if (EC)
+ return createFileError(Path, EC);
+
+ llvm::sort(FoundDBDirs, [](const DBDir &LHS, const DBDir &RHS) -> bool {
+ return LHS.Order <= RHS.Order;
+ });
+ for (DBDir &Dir : FoundDBDirs)
+ DBDirs.push_back(std::move(Dir.Name));
+ return Error::success();
+}
+
+static Error getAllGarbageDirs(StringRef Path,
+ SmallVectorImpl<std::string> &DBDirs) {
+ if (Error E = getAllDBDirs(Path, DBDirs, /*IncludeCorrupt=*/true))
+ return E;
+
+ // FIXME: When the version of \p DBDirPrefix is bumped up we need to figure
+ // out how to handle the leftover sub-directories of the previous version.
+
+ for (unsigned Keep = 2; Keep > 0 && !DBDirs.empty(); --Keep) {
+ StringRef Back(DBDirs.back());
+ if (Back.starts_with(CorruptPrefix))
+ break;
+ DBDirs.pop_back();
+ }
+ return Error::success();
+}
+
+/// \returns Given a sub-directory named 'v<version>.<x>', it outputs the
+/// 'v<version>.<x+1>' name.
+static void getNextDBDirName(StringRef DBDir, llvm::raw_ostream &OS) {
+ assert(DBDir.starts_with(DBDirPrefix));
+ uint64_t Count;
+ bool Failed = DBDir.substr(DBDirPrefix.size()).getAsInteger(10, Count);
+ assert(!Failed);
+ (void)Failed;
+ OS << DBDirPrefix << Count + 1;
+}
+
+static Error validateOutOfProcess(StringRef LLVMCasBinary, StringRef RootPath,
+ bool CheckHash) {
+ SmallVector<StringRef> Args{LLVMCasBinary, "-cas", RootPath, "-validate"};
+ if (CheckHash)
+ Args.push_back("-check-hash");
+
+ llvm::SmallString<128> StdErrPath;
+ int StdErrFD = -1;
+ if (std::error_code EC = sys::fs::createTemporaryFile(
+ "llvm-cas-validate-stderr", "txt", StdErrFD, StdErrPath,
+ llvm::sys::fs::OF_Text))
+ return createStringError(EC, "failed to create temporary file");
+ FileRemover OutputRemover(StdErrPath.c_str());
+
+ std::optional<llvm::StringRef> Redirects[] = {
+ {""}, // stdin = /dev/null
+ {""}, // stdout = /dev/null
+ StdErrPath.str(),
+ };
+
+ std::string ErrMsg;
+ int Result =
+ sys::ExecuteAndWait(LLVMCasBinary, Args, /*Env=*/std::nullopt, Redirects,
+ /*SecondsToWait=*/120, /*MemoryLimit=*/0, &ErrMsg);
+
+ if (Result == -1)
+ return createStringError("failed to exec " + join(Args, " ") + ": " +
+ ErrMsg);
+ if (Result != 0) {
+ llvm::SmallString<64> Err("cas contents invalid");
+ if (!ErrMsg.empty()) {
+ Err += ": ";
+ Err += ErrMsg;
+ }
+ auto StdErrBuf = MemoryBuffer::getFile(StdErrPath.c_str());
+ if (StdErrBuf && !(*StdErrBuf)->getBuffer().empty()) {
+ Err += ": ";
+ Err += (*StdErrBuf)->getBuffer();
+ }
+ return createStringError(Err);
+ }
+ return Error::success();
+}
+
+static Error validateInProcess(StringRef RootPath, StringRef HashName,
+ unsigned HashByteSize, bool CheckHash) {
+ std::shared_ptr<UnifiedOnDiskCache> UniDB;
+ if (Error E = UnifiedOnDiskCache::open(RootPath, std::nullopt, HashName,
+ HashByteSize)
+ .moveInto(UniDB))
+ return E;
+ auto CAS = builtin::createObjectStoreFromUnifiedOnDiskCache(UniDB);
+ if (Error E = CAS->validate(CheckHash))
+ return E;
+ auto Cache = builtin::createActionCacheFromUnifiedOnDiskCache(UniDB);
+ if (Error E = Cache->validate())
+ return E;
+ return Error::success();
+}
+
+static Expected<uint64_t> getBootTime() {
+#if __has_include(<sys/sysctl.h>) && defined(KERN_BOOTTIME)
+ struct timeval TV;
+ size_t TVLen = sizeof(TV);
+ int KernBoot[2] = {CTL_KERN, KERN_BOOTTIME};
+ if (sysctl(KernBoot, 2, &TV, &TVLen, nullptr, 0) < 0)
+ return createStringError(llvm::errnoAsErrorCode(),
+ "failed to get boottime");
+ if (TVLen != sizeof(TV))
+ return createStringError("sysctl kern.boottime unexpected format");
+ return TV.tv_sec;
+#elif defined(__linux__)
+ // Use the mtime for /proc, which is recreated during system boot.
+ // We could also read /proc/stat and search for 'btime'.
+ sys::fs::file_status Status;
+ if (std::error_code EC = sys::fs::status("/proc", Status))
+ return createFileError("/proc", EC);
+ return Status.getLastModificationTime().time_since_epoch().count();
+#else
+ llvm::report_fatal_error("unimplemented");
----------------
bogner wrote:
Better breadcrumb:
```suggestion
llvm::report_fatal_error("getBootTime unimplemented");
```
https://github.com/llvm/llvm-project/pull/114103
More information about the llvm-commits
mailing list