[llvm] f9cd2ee - [CAS] Add MappedFileRegionArena (#114099)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 15 10:00:25 PDT 2025
Author: Steven Wu
Date: 2025-09-15T10:00:21-07:00
New Revision: f9cd2ee119ff6bd59f48fd71617dc4fc981083c9
URL: https://github.com/llvm/llvm-project/commit/f9cd2ee119ff6bd59f48fd71617dc4fc981083c9
DIFF: https://github.com/llvm/llvm-project/commit/f9cd2ee119ff6bd59f48fd71617dc4fc981083c9.diff
LOG: [CAS] Add MappedFileRegionArena (#114099)
Add MappedFileRegionArena which can be served as a file system backed
persistent memory allocator. The allocator works like a
BumpPtrAllocator,
and is designed to be thread safe and process safe.
The implementation relies on the POSIX compliance of file system and
doesn't work on all file systems. If the file system supports lazy tail
(doesn't allocate disk space if the tail of the large file is not used),
user has more flexibility to declare a larger capacity.
The allocator works by using a atomically updated bump ptr at a location
that can be customized by the user. The atomic pointer points to the
next available space to allocate, and the allocator will resize/truncate
to current usage once all clients closed the allocator.
Windows implementation contributed by: @hjyamauchi
Added:
llvm/include/llvm/CAS/MappedFileRegionArena.h
llvm/lib/CAS/MappedFileRegionArena.cpp
llvm/lib/CAS/OnDiskCommon.cpp
llvm/lib/CAS/OnDiskCommon.h
llvm/unittests/CAS/ProgramTest.cpp
Modified:
llvm/CMakeLists.txt
llvm/include/llvm/Config/llvm-config.h.cmake
llvm/include/llvm/Support/FileSystem.h
llvm/lib/CAS/CMakeLists.txt
llvm/lib/Support/Unix/Path.inc
llvm/lib/Support/Windows/Path.inc
llvm/unittests/CAS/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index b98192968a3ab..e8af7fb432f40 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -878,6 +878,7 @@ option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF)
option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON)
option (LLVM_ENABLE_BINDINGS "Build bindings." ON)
option (LLVM_ENABLE_TELEMETRY "Enable the telemetry library. If set to OFF, library cannot be enabled after build (eg., at runtime)" ON)
+option (LLVM_ENABLE_ONDISK_CAS "Build OnDiskCAS." ON)
set(LLVM_INSTALL_DOXYGEN_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/llvm/doxygen-html"
CACHE STRING "Doxygen-generated HTML documentation install directory")
diff --git a/llvm/include/llvm/CAS/MappedFileRegionArena.h b/llvm/include/llvm/CAS/MappedFileRegionArena.h
new file mode 100644
index 0000000000000..ff51f0eb59929
--- /dev/null
+++ b/llvm/include/llvm/CAS/MappedFileRegionArena.h
@@ -0,0 +1,130 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file declares interface for MappedFileRegionArena, a bump pointer
+/// allocator, backed by a memory-mapped file.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_MAPPEDFILEREGIONARENA_H
+#define LLVM_CAS_MAPPEDFILEREGIONARENA_H
+
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/FileSystem.h"
+#include <atomic>
+
+namespace llvm::cas {
+
+/// Allocator for an owned mapped file region that supports thread-safe and
+/// process-safe bump pointer allocation.
+///
+/// This allocator is designed to create a sparse file when supported by the
+/// filesystem's \c ftruncate so that it can be used with a large maximum size.
+/// It will also attempt to shrink the underlying file down to its current
+/// allocation size when the last concurrent mapping is closed.
+///
+/// Process-safe. Uses file locks when resizing the file during initialization
+/// and destruction.
+///
+/// Thread-safe. Requires OS support thread-safe file lock.
+///
+/// Provides 8-byte alignment for all allocations.
+class MappedFileRegionArena {
+public:
+ using RegionT = sys::fs::mapped_file_region;
+
+ /// Header for MappedFileRegionArena. It can be configured to be located
+ /// at any location within the file and the allocation will be appended after
+ /// the header.
+ struct Header {
+ // BumpPtr for new allocation.
+ std::atomic<uint64_t> BumpPtr;
+ // Allocated size on disk.
+ std::atomic<uint64_t> AllocatedSize;
+ // Capacity of the file.
+ std::atomic<uint64_t> Capacity;
+ // Offset from the beginning of the file to this header (for verification).
+ std::atomic<uint64_t> HeaderOffset;
+ };
+
+ /// Create a \c MappedFileRegionArena.
+ ///
+ /// \param Path the path to open the mapped region.
+ /// \param Capacity the maximum size for the mapped file region.
+ /// \param HeaderOffset the offset at which to store the header. This is so
+ /// that information can be stored before the header, like a file magic.
+ /// \param NewFileConstructor is for constructing new files. It has exclusive
+ /// access to the file. Must call \c initializeBumpPtr.
+ static Expected<MappedFileRegionArena>
+ create(const Twine &Path, uint64_t Capacity, uint64_t HeaderOffset,
+ function_ref<Error(MappedFileRegionArena &)> NewFileConstructor);
+
+ /// Minimum alignment for allocations, currently hardcoded to 8B.
+ static constexpr Align getAlign() {
+ // Trick Align into giving us '8' as a constexpr.
+ struct alignas(8) T {};
+ static_assert(alignof(T) == 8, "Tautology failed?");
+ return Align::Of<T>();
+ }
+
+ /// Allocate at least \p AllocSize. Rounds up to \a getAlign().
+ Expected<char *> allocate(uint64_t AllocSize) {
+ auto Offset = allocateOffset(AllocSize);
+ if (LLVM_UNLIKELY(!Offset))
+ return Offset.takeError();
+ return data() + *Offset;
+ }
+ /// Allocate, returning the offset from \a data() instead of a pointer.
+ Expected<int64_t> allocateOffset(uint64_t AllocSize);
+
+ char *data() const { return Region.data(); }
+ uint64_t size() const { return H->BumpPtr; }
+ uint64_t capacity() const { return Region.size(); }
+
+ RegionT &getRegion() { return Region; }
+
+ ~MappedFileRegionArena() { destroyImpl(); }
+
+ MappedFileRegionArena() = default;
+ MappedFileRegionArena(MappedFileRegionArena &&RHS) { moveImpl(RHS); }
+ MappedFileRegionArena &operator=(MappedFileRegionArena &&RHS) {
+ destroyImpl();
+ moveImpl(RHS);
+ return *this;
+ }
+
+ MappedFileRegionArena(const MappedFileRegionArena &) = delete;
+ MappedFileRegionArena &operator=(const MappedFileRegionArena &) = delete;
+
+private:
+ // initialize header from offset.
+ void initializeHeader(uint64_t HeaderOffset);
+
+ void destroyImpl();
+ void moveImpl(MappedFileRegionArena &RHS) {
+ std::swap(Region, RHS.Region);
+ std::swap(H, RHS.H);
+ std::swap(Path, RHS.Path);
+ std::swap(FD, RHS.FD);
+ std::swap(SharedLockFD, RHS.SharedLockFD);
+ }
+
+private:
+ RegionT Region;
+ Header *H = nullptr;
+ std::string Path;
+ // File descriptor for the main storage file.
+ std::optional<int> FD;
+ // File descriptor for the file used as reader/writer lock.
+ std::optional<int> SharedLockFD;
+};
+
+} // namespace llvm::cas
+
+#endif // LLVM_CAS_MAPPEDFILEREGIONARENA_H
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 39136bc45c292..6488d6c01b5c6 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -146,4 +146,7 @@
coverage bugs, and to 0 otherwise. */
#cmakedefine01 LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+/* Define to 1 to enable LLVM OnDisk Content Addressable Storage */
+#cmakedefine01 LLVM_ENABLE_ONDISK_CAS
+
#endif
diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index a21b0a272d2b0..c203779307840 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -410,6 +410,11 @@ LLVM_ABI std::error_code copy_file(const Twine &From, int ToFD);
/// platform-specific error_code.
LLVM_ABI std::error_code resize_file(int FD, uint64_t Size);
+/// Resize path to size with sparse files explicitly enabled. It uses
+/// FSCTL_SET_SPARSE On Windows. This is the same as resize_file on
+/// non-Windows
+LLVM_ABI std::error_code resize_file_sparse(int FD, uint64_t Size);
+
/// Resize \p FD to \p Size before mapping \a mapped_file_region::readwrite. On
/// non-Windows, this calls \a resize_file(). On Windows, this is a no-op,
/// since the subsequent mapping (via \c CreateFileMapping) automatically
diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt
index f3d2b41c704bc..6ed724bc2fd76 100644
--- a/llvm/lib/CAS/CMakeLists.txt
+++ b/llvm/lib/CAS/CMakeLists.txt
@@ -3,7 +3,9 @@ add_llvm_component_library(LLVMCAS
ActionCaches.cpp
BuiltinCAS.cpp
InMemoryCAS.cpp
+ MappedFileRegionArena.cpp
ObjectStore.cpp
+ OnDiskCommon.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/CAS
diff --git a/llvm/lib/CAS/MappedFileRegionArena.cpp b/llvm/lib/CAS/MappedFileRegionArena.cpp
new file mode 100644
index 0000000000000..3c920edcaae6a
--- /dev/null
+++ b/llvm/lib/CAS/MappedFileRegionArena.cpp
@@ -0,0 +1,388 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file Implements MappedFileRegionArena.
+///
+/// A bump pointer allocator, backed by a memory-mapped file.
+///
+/// The effect we want is:
+///
+/// Step 1. If it doesn't exist, create the file with an initial size.
+/// Step 2. Reserve virtual memory large enough for the max file size.
+/// Step 3. Map the file into memory in the reserved region.
+/// Step 4. Increase the file size and update the mapping when necessary.
+///
+/// However, updating the mapping is challenging when it needs to work portably,
+/// and across multiple processes without locking for every read. Our current
+/// implementation handles the steps above in following ways:
+///
+/// Step 1. Use \ref sys::fs::resize_file_sparse to grow the file to its max
+/// size (typically several GB). If the file system doesn't support
+/// sparse file, this may return a fully allocated file.
+/// Step 2. Call \ref sys::fs::mapped_file_region to map the entire file.
+/// Step 3. [Automatic as part of step 2.]
+/// Step 4. If supported, use \c fallocate or similiar APIs to ensure the file
+/// system storage for the sparse file so we won't end up with partial
+/// file if the disk is out of space.
+///
+/// Additionally, we attempt to resize the file to its actual data size when
+/// closing the mapping, if this is the only concurrent instance. This is done
+/// using file locks. Shrinking the file mitigates problems with having large
+/// files: on filesystems without sparse files it avoids unnecessary space use;
+/// it also avoids allocating the full size if another process copies the file,
+/// which typically loses sparseness. These mitigations only work while the file
+/// is not in use.
+///
+/// The capacity and the header offset is determined by the first user of the
+/// MappedFileRegionArena instance and any future mismatched value from the
+/// original will result in error on creation.
+///
+/// To support resizing, we use two separate file locks:
+/// 1. We use a shared reader lock on a ".shared" file until destruction.
+/// 2. We use a lock on the main file during initialization - shared to check
+/// the status, upgraded to exclusive to resize/initialize the file.
+///
+/// Then during destruction we attempt to get exclusive access on (1), which
+/// requires no concurrent readers. If so, we shrink the file. Using two
+/// separate locks simplifies the implementation and enables it to work on
+/// platforms (e.g. Windows) where a shared/reader lock prevents writing.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/MappedFileRegionArena.h"
+#include "OnDiskCommon.h"
+#include "llvm/ADT/StringExtras.h"
+
+#if LLVM_ON_UNIX
+#include <sys/stat.h>
+#if __has_include(<sys/param.h>)
+#include <sys/param.h>
+#endif
+#ifdef DEV_BSIZE
+#define MAPPED_FILE_BSIZE DEV_BSIZE
+#elif __linux__
+#define MAPPED_FILE_BSIZE 512
+#endif
+#endif
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::ondisk;
+
+namespace {
+struct FileWithLock {
+ std::string Path;
+ int FD = -1;
+ std::optional<sys::fs::LockKind> Locked;
+
+private:
+ FileWithLock(std::string PathStr, Error &E) : Path(std::move(PathStr)) {
+ ErrorAsOutParameter EOP(&E);
+ if (std::error_code EC = sys::fs::openFileForReadWrite(
+ Path, FD, sys::fs::CD_OpenAlways, sys::fs::OF_None))
+ E = createFileError(Path, EC);
+ }
+
+public:
+ FileWithLock(FileWithLock &) = delete;
+ FileWithLock(FileWithLock &&Other) {
+ Path = std::move(Other.Path);
+ FD = Other.FD;
+ Other.FD = -1;
+ Locked = Other.Locked;
+ Other.Locked = std::nullopt;
+ }
+
+ ~FileWithLock() { consumeError(unlock()); }
+
+ static Expected<FileWithLock> open(StringRef Path) {
+ Error E = Error::success();
+ FileWithLock Result(Path.str(), E);
+ if (E)
+ return std::move(E);
+ return std::move(Result);
+ }
+
+ Error lock(sys::fs::LockKind LK) {
+ assert(!Locked && "already locked");
+ if (std::error_code EC = lockFileThreadSafe(FD, LK))
+ return createFileError(Path, EC);
+ Locked = LK;
+ return Error::success();
+ }
+
+ Error switchLock(sys::fs::LockKind LK) {
+ assert(Locked && "not locked");
+ if (auto E = unlock())
+ return E;
+
+ return lock(LK);
+ }
+
+ Error unlock() {
+ if (Locked) {
+ Locked = std::nullopt;
+ if (std::error_code EC = unlockFileThreadSafe(FD))
+ return createFileError(Path, EC);
+ }
+ return Error::success();
+ }
+
+ // Return true if succeed to lock the file exclusively.
+ bool tryLockExclusive() {
+ assert(!Locked && "can only try to lock if not locked");
+ if (tryLockFileThreadSafe(FD) == std::error_code()) {
+ Locked = sys::fs::LockKind::Exclusive;
+ return true;
+ }
+
+ return false;
+ }
+
+ // Release the lock so it will not be unlocked on destruction.
+ void release() {
+ Locked = std::nullopt;
+ FD = -1;
+ }
+};
+
+struct FileSizeInfo {
+ uint64_t Size;
+ uint64_t AllocatedSize;
+
+ static ErrorOr<FileSizeInfo> get(sys::fs::file_t File);
+};
+} // end anonymous namespace
+
+Expected<MappedFileRegionArena> MappedFileRegionArena::create(
+ const Twine &Path, uint64_t Capacity, uint64_t HeaderOffset,
+ function_ref<Error(MappedFileRegionArena &)> NewFileConstructor) {
+ uint64_t MinCapacity = HeaderOffset + sizeof(Header);
+ if (Capacity < MinCapacity)
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ "capacity is too small to hold MappedFileRegionArena");
+
+ MappedFileRegionArena Result;
+ Result.Path = Path.str();
+
+ // Open the shared lock file. See file comment for details of locking scheme.
+ SmallString<128> SharedFilePath(Result.Path);
+ SharedFilePath.append(".shared");
+
+ auto SharedFileLock = FileWithLock::open(SharedFilePath);
+ if (!SharedFileLock)
+ return SharedFileLock.takeError();
+ Result.SharedLockFD = SharedFileLock->FD;
+
+ // Take shared/reader lock that will be held until destroyImpl if construction
+ // is successful.
+ if (auto E = SharedFileLock->lock(sys::fs::LockKind::Shared))
+ return std::move(E);
+
+ // Take shared/reader lock for initialization.
+ auto MainFile = FileWithLock::open(Result.Path);
+ if (!MainFile)
+ return MainFile.takeError();
+ if (Error E = MainFile->lock(sys::fs::LockKind::Shared))
+ return std::move(E);
+ Result.FD = MainFile->FD;
+
+ sys::fs::file_t File = sys::fs::convertFDToNativeFile(MainFile->FD);
+ auto FileSize = FileSizeInfo::get(File);
+ if (!FileSize)
+ return createFileError(Result.Path, FileSize.getError());
+
+ // If the size is smaller than the capacity, we need to initialize the file.
+ // It maybe empty, or may have been shrunk during a previous close.
+ if (FileSize->Size < Capacity) {
+ // Lock the file exclusively so only one process will do the initialization.
+ if (Error E = MainFile->switchLock(sys::fs::LockKind::Exclusive))
+ return std::move(E);
+ // Retrieve the current size now that we have exclusive access.
+ FileSize = FileSizeInfo::get(File);
+ if (!FileSize)
+ return createFileError(Result.Path, FileSize.getError());
+ }
+
+ if (FileSize->Size >= MinCapacity) {
+ // File is initialized. Read out the header to check for capacity and
+ // offset.
+ SmallVector<char, sizeof(Header)> HeaderContent(sizeof(Header));
+ auto Size = sys::fs::readNativeFileSlice(File, HeaderContent, HeaderOffset);
+ if (!Size)
+ return Size.takeError();
+
+ Header *H = reinterpret_cast<Header *>(HeaderContent.data());
+ if (H->HeaderOffset != HeaderOffset)
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ "specified header offset (" + utostr(HeaderOffset) +
+ ") does not match existing config (" + utostr(H->HeaderOffset) +
+ ")");
+
+ // If the capacity doesn't match, use the existing capacity instead.
+ if (H->Capacity != Capacity)
+ Capacity = H->Capacity;
+ }
+
+ // If the size is smaller than capacity, we need to resize the file.
+ if (FileSize->Size < Capacity) {
+ assert(MainFile->Locked == sys::fs::LockKind::Exclusive);
+ if (std::error_code EC =
+ sys::fs::resize_file_sparse(MainFile->FD, Capacity))
+ return createFileError(Result.Path, EC);
+ }
+
+ // Create the mapped region.
+ {
+ std::error_code EC;
+ sys::fs::mapped_file_region Map(
+ File, sys::fs::mapped_file_region::readwrite, Capacity, 0, EC);
+ if (EC)
+ return createFileError(Result.Path, EC);
+ Result.Region = std::move(Map);
+ }
+
+ // Initialize the header.
+ Result.initializeHeader(HeaderOffset);
+ if (FileSize->Size < MinCapacity) {
+ assert(MainFile->Locked == sys::fs::LockKind::Exclusive);
+ // If we need to fully initialize the file, call NewFileConstructor.
+ if (Error E = NewFileConstructor(Result))
+ return std::move(E);
+
+ Result.H->HeaderOffset.exchange(HeaderOffset);
+ Result.H->Capacity.exchange(Capacity);
+ }
+
+ if (MainFile->Locked == sys::fs::LockKind::Exclusive) {
+ // If holding an exclusive lock, we might have resized the file and
+ // performed some read/write to the file. Query the file size again to make
+ // sure everything is up-to-date. Otherwise, FileSize info is already
+ // up-to-date.
+ FileSize = FileSizeInfo::get(File);
+ if (!FileSize)
+ return createFileError(Result.Path, FileSize.getError());
+ Result.H->AllocatedSize.exchange(FileSize->AllocatedSize);
+ }
+
+ // Release the shared lock so it can be closed in destoryImpl().
+ SharedFileLock->release();
+ return Result;
+}
+
+void MappedFileRegionArena::destroyImpl() {
+ if (!FD)
+ return;
+
+ // Drop the shared lock indicating we are no longer accessing the file.
+ if (SharedLockFD)
+ (void)unlockFileThreadSafe(*SharedLockFD);
+
+ // Attempt to truncate the file if we can get exclusive access. Ignore any
+ // errors.
+ if (H) {
+ assert(SharedLockFD && "Must have shared lock file open");
+ if (tryLockFileThreadSafe(*SharedLockFD) == std::error_code()) {
+ size_t Size = size();
+ // sync to file system to make sure all contents are up-to-date.
+ (void)Region.sync();
+ // unmap the file before resizing since that is the requirement for
+ // some platforms.
+ Region.unmap();
+ (void)sys::fs::resize_file(*FD, Size);
+ (void)unlockFileThreadSafe(*SharedLockFD);
+ }
+ }
+
+ auto Close = [](std::optional<int> &FD) {
+ if (FD) {
+ sys::fs::file_t File = sys::fs::convertFDToNativeFile(*FD);
+ sys::fs::closeFile(File);
+ FD = std::nullopt;
+ }
+ };
+
+ // Close the file and shared lock.
+ Close(FD);
+ Close(SharedLockFD);
+}
+
+void MappedFileRegionArena::initializeHeader(uint64_t HeaderOffset) {
+ assert(capacity() < (uint64_t)INT64_MAX && "capacity must fit in int64_t");
+ uint64_t HeaderEndOffset = HeaderOffset + sizeof(decltype(*H));
+ assert(HeaderEndOffset <= capacity() &&
+ "Expected end offset to be pre-allocated");
+ assert(isAligned(Align::Of<decltype(*H)>(), HeaderOffset) &&
+ "Expected end offset to be aligned");
+ H = reinterpret_cast<decltype(H)>(data() + HeaderOffset);
+
+ uint64_t ExistingValue = 0;
+ if (!H->BumpPtr.compare_exchange_strong(ExistingValue, HeaderEndOffset))
+ assert(ExistingValue >= HeaderEndOffset &&
+ "Expected 0, or past the end of the header itself");
+}
+
+static Error createAllocatorOutOfSpaceError() {
+ return createStringError(std::make_error_code(std::errc::not_enough_memory),
+ "memory mapped file allocator is out of space");
+}
+
+Expected<int64_t> MappedFileRegionArena::allocateOffset(uint64_t AllocSize) {
+ AllocSize = alignTo(AllocSize, getAlign());
+ uint64_t OldEnd = H->BumpPtr.fetch_add(AllocSize);
+ uint64_t NewEnd = OldEnd + AllocSize;
+ if (LLVM_UNLIKELY(NewEnd > capacity())) {
+ // Return the allocation. If the start already passed the end, that means
+ // some other concurrent allocations already consumed all the capacity.
+ // There is no need to return the original value. If the start was not
+ // passed the end, current allocation certainly bumped it passed the end.
+ // All other allocation afterwards must have failed and current allocation
+ // is in charge of return the allocation back to a valid value.
+ if (OldEnd <= capacity())
+ (void)H->BumpPtr.exchange(OldEnd);
+
+ return createAllocatorOutOfSpaceError();
+ }
+
+ uint64_t DiskSize = H->AllocatedSize;
+ if (LLVM_UNLIKELY(NewEnd > DiskSize)) {
+ uint64_t NewSize;
+ // The minimum increment is a page, but allocate more to amortize the cost.
+ constexpr uint64_t Increment = 1 * 1024 * 1024; // 1 MB
+ if (Error E = preallocateFileTail(*FD, DiskSize, DiskSize + Increment)
+ .moveInto(NewSize))
+ return std::move(E);
+ assert(NewSize >= DiskSize + Increment);
+ // FIXME: on Darwin this can under-count the size if there is a race to
+ // preallocate disk, because the semantics of F_PREALLOCATE are to add bytes
+ // to the end of the file, not to allocate up to a fixed size.
+ // Any discrepancy will be resolved the next time the file is truncated and
+ // then reopend.
+ while (DiskSize < NewSize)
+ H->AllocatedSize.compare_exchange_strong(DiskSize, NewSize);
+ }
+ return OldEnd;
+}
+
+ErrorOr<FileSizeInfo> FileSizeInfo::get(sys::fs::file_t File) {
+#if LLVM_ON_UNIX && defined(MAPPED_FILE_BSIZE)
+ struct stat Status;
+ int StatRet = ::fstat(File, &Status);
+ if (StatRet)
+ return errnoAsErrorCode();
+ uint64_t AllocatedSize = uint64_t(Status.st_blksize) * MAPPED_FILE_BSIZE;
+ return FileSizeInfo{uint64_t(Status.st_size), AllocatedSize};
+#else
+ // Fallback: assume the file is fully allocated. Note: this may result in
+ // data loss on out-of-space.
+ sys::fs::file_status Status;
+ if (std::error_code EC = sys::fs::status(File, Status))
+ return EC;
+ return FileSizeInfo{Status.getSize(), Status.getSize()};
+#endif
+}
diff --git a/llvm/lib/CAS/OnDiskCommon.cpp b/llvm/lib/CAS/OnDiskCommon.cpp
new file mode 100644
index 0000000000000..f7765c58944c5
--- /dev/null
+++ b/llvm/lib/CAS/OnDiskCommon.cpp
@@ -0,0 +1,121 @@
+//===- OnDiskCommon.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OnDiskCommon.h"
+#include "llvm/Config/config.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include <thread>
+
+#if __has_include(<sys/file.h>)
+#include <sys/file.h>
+#ifdef LOCK_SH
+#define HAVE_FLOCK 1
+#else
+#define HAVE_FLOCK 0
+#endif
+#endif
+
+#if __has_include(<fcntl.h>)
+#include <fcntl.h>
+#endif
+
+using namespace llvm;
+
+std::error_code cas::ondisk::lockFileThreadSafe(int FD,
+ sys::fs::LockKind Kind) {
+#if HAVE_FLOCK
+ if (flock(FD, Kind == sys::fs::LockKind::Exclusive ? LOCK_EX : LOCK_SH) == 0)
+ return std::error_code();
+ return std::error_code(errno, std::generic_category());
+#elif defined(_WIN32)
+ // On Windows this implementation is thread-safe.
+ return sys::fs::lockFile(FD, Kind);
+#else
+ return make_error_code(std::errc::no_lock_available);
+#endif
+}
+
+std::error_code cas::ondisk::unlockFileThreadSafe(int FD) {
+#if HAVE_FLOCK
+ if (flock(FD, LOCK_UN) == 0)
+ return std::error_code();
+ return std::error_code(errno, std::generic_category());
+#elif defined(_WIN32)
+ // On Windows this implementation is thread-safe.
+ return sys::fs::unlockFile(FD);
+#else
+ return make_error_code(std::errc::no_lock_available);
+#endif
+}
+
+std::error_code
+cas::ondisk::tryLockFileThreadSafe(int FD, std::chrono::milliseconds Timeout,
+ sys::fs::LockKind Kind) {
+#if HAVE_FLOCK
+ auto Start = std::chrono::steady_clock::now();
+ auto End = Start + Timeout;
+ do {
+ if (flock(FD, (Kind == sys::fs::LockKind::Exclusive ? LOCK_EX : LOCK_SH) |
+ LOCK_NB) == 0)
+ return std::error_code();
+ int Error = errno;
+ if (Error == EWOULDBLOCK) {
+ // Match sys::fs::tryLockFile, which sleeps for 1 ms per attempt.
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ continue;
+ }
+ return std::error_code(Error, std::generic_category());
+ } while (std::chrono::steady_clock::now() < End);
+ return make_error_code(std::errc::no_lock_available);
+#elif defined(_WIN32)
+ // On Windows this implementation is thread-safe.
+ return sys::fs::tryLockFile(FD, Timeout, Kind);
+#else
+ return make_error_code(std::errc::no_lock_available);
+#endif
+}
+
+Expected<size_t> cas::ondisk::preallocateFileTail(int FD, size_t CurrentSize,
+ size_t NewSize) {
+ auto CreateError = [&](std::error_code EC) -> Expected<size_t> {
+ if (EC == std::errc::not_supported)
+ // Ignore ENOTSUP in case the filesystem cannot preallocate.
+ return NewSize;
+#if defined(HAVE_POSIX_FALLOCATE)
+ if (EC == std::errc::invalid_argument && CurrentSize < NewSize && // len > 0
+ NewSize < std::numeric_limits<off_t>::max()) // 0 <= offset, len < max
+ // Prior to 2024, POSIX required EINVAL for cases that should be ENOTSUP,
+ // so handle it the same as above if it is not one of the other ways to
+ // get EINVAL.
+ return NewSize;
+#endif
+ return createStringError(EC,
+ "failed to allocate to CAS file: " + EC.message());
+ };
+#if defined(HAVE_POSIX_FALLOCATE)
+ // Note: posix_fallocate returns its error directly, not via errno.
+ if (int Err = posix_fallocate(FD, CurrentSize, NewSize - CurrentSize))
+ return CreateError(std::error_code(Err, std::generic_category()));
+ return NewSize;
+#elif defined(__APPLE__)
+ fstore_t FAlloc;
+ FAlloc.fst_flags = F_ALLOCATEALL | F_ALLOCATEPERSIST;
+ FAlloc.fst_posmode = F_PEOFPOSMODE;
+ FAlloc.fst_offset = 0;
+ FAlloc.fst_length = NewSize - CurrentSize;
+ FAlloc.fst_bytesalloc = 0;
+ if (fcntl(FD, F_PREALLOCATE, &FAlloc))
+ return CreateError(errnoAsErrorCode());
+ assert(CurrentSize + FAlloc.fst_bytesalloc >= NewSize);
+ return CurrentSize + FAlloc.fst_bytesalloc;
+#else
+ (void)CreateError; // Silence unused variable.
+ return NewSize; // Pretend it worked.
+#endif
+}
diff --git a/llvm/lib/CAS/OnDiskCommon.h b/llvm/lib/CAS/OnDiskCommon.h
new file mode 100644
index 0000000000000..8b79ffe5c3158
--- /dev/null
+++ b/llvm/lib/CAS/OnDiskCommon.h
@@ -0,0 +1,46 @@
+//===- OnDiskCommon.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CAS_ONDISKCOMMON_H
+#define LLVM_LIB_CAS_ONDISKCOMMON_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include <chrono>
+
+namespace llvm::cas::ondisk {
+
+/// Thread-safe alternative to \c sys::fs::lockFile. This does not support all
+/// the platforms that \c sys::fs::lockFile does, so keep it in the CAS library
+/// for now.
+std::error_code lockFileThreadSafe(int FD, llvm::sys::fs::LockKind Kind);
+
+/// Thread-safe alternative to \c sys::fs::unlockFile. This does not support all
+/// the platforms that \c sys::fs::lockFile does, so keep it in the CAS library
+/// for now.
+std::error_code unlockFileThreadSafe(int FD);
+
+/// Thread-safe alternative to \c sys::fs::tryLockFile. This does not support
+/// all the platforms that \c sys::fs::lockFile does, so keep it in the CAS
+/// library for now.
+std::error_code tryLockFileThreadSafe(
+ int FD, std::chrono::milliseconds Timeout = std::chrono::milliseconds(0),
+ llvm::sys::fs::LockKind Kind = llvm::sys::fs::LockKind::Exclusive);
+
+/// Allocate space for the file \p FD on disk, if the filesystem supports it.
+///
+/// On filesystems that support this operation, this ensures errors such as
+/// \c std::errc::no_space_on_device are detected before we write data.
+///
+/// \returns the new size of the file, or an \c Error.
+Expected<size_t> preallocateFileTail(int FD, size_t CurrentSize,
+ size_t NewSize);
+
+} // namespace llvm::cas::ondisk
+
+#endif // LLVM_LIB_CAS_ONDISKCOMMON_H
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index 2f563e2899b56..0d991ead72416 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -600,6 +600,11 @@ std::error_code resize_file(int FD, uint64_t Size) {
return std::error_code();
}
+std::error_code resize_file_sparse(int FD, uint64_t Size) {
+ // On Unix, this is the same as `resize_file`.
+ return resize_file(FD, Size);
+}
+
static int convertAccessMode(AccessMode Mode) {
switch (Mode) {
case AccessMode::Exist:
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index 6672d8e0ec777..be007b7abdb51 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -27,6 +27,7 @@
#include "llvm/Support/Windows/WindowsSupport.h"
#include <shellapi.h>
#include <shlobj.h>
+#include <winioctl.h>
#undef max
@@ -617,6 +618,22 @@ std::error_code resize_file(int FD, uint64_t Size) {
return std::error_code(error, std::generic_category());
}
+std::error_code resize_file_sparse(int FD, uint64_t Size) {
+ HANDLE hFile = reinterpret_cast<HANDLE>(::_get_osfhandle(FD));
+ DWORD temp;
+ if (!DeviceIoControl(hFile, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &temp,
+ NULL)) {
+ return mapWindowsError(GetLastError());
+ }
+ LARGE_INTEGER liSize;
+ liSize.QuadPart = Size;
+ if (!SetFilePointerEx(hFile, liSize, NULL, FILE_BEGIN) ||
+ !SetEndOfFile(hFile)) {
+ return mapWindowsError(GetLastError());
+ }
+ return std::error_code();
+}
+
std::error_code access(const Twine &Path, AccessMode Mode) {
SmallVector<wchar_t, 128> PathUtf16;
diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt
index ff081007f31bc..ab709e30369bf 100644
--- a/llvm/unittests/CAS/CMakeLists.txt
+++ b/llvm/unittests/CAS/CMakeLists.txt
@@ -1,3 +1,7 @@
+if (LLVM_ENABLE_ONDISK_CAS)
+ add_definitions(-DLLVM_ENABLE_ONDISK_CAS=1)
+endif()
+
set(LLVM_LINK_COMPONENTS
Support
CAS
@@ -8,6 +12,7 @@ add_llvm_unittest(CASTests
ActionCacheTest.cpp
CASTestConfig.cpp
ObjectStoreTest.cpp
+ ProgramTest.cpp
)
target_link_libraries(CASTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/CAS/ProgramTest.cpp b/llvm/unittests/CAS/ProgramTest.cpp
new file mode 100644
index 0000000000000..3ba5f697eeed5
--- /dev/null
+++ b/llvm/unittests/CAS/ProgramTest.cpp
@@ -0,0 +1,239 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Program.h"
+#include "llvm/CAS/MappedFileRegionArena.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/ExponentialBackoff.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+#if defined(__APPLE__)
+#include <crt_externs.h>
+#elif !defined(_MSC_VER)
+// Forward declare environ in case it's not provided by stdlib.h.
+extern char **environ;
+#endif
+
+using namespace llvm;
+using namespace llvm::cas;
+
+extern const char *TestMainArgv0;
+static char ProgramID = 0;
+
+class CASProgramTest : public testing::Test {
+ std::vector<StringRef> EnvTable;
+ std::vector<std::string> EnvStorage;
+
+protected:
+ void SetUp() override {
+ auto EnvP = [] {
+#if defined(_WIN32)
+ _wgetenv(L"TMP"); // Populate _wenviron, initially is null
+ return _wenviron;
+#elif defined(__APPLE__)
+ return *_NSGetEnviron();
+#else
+ return environ;
+#endif
+ }();
+ ASSERT_TRUE(EnvP);
+
+ auto prepareEnvVar = [this](decltype(*EnvP) Var) -> StringRef {
+#if defined(_WIN32)
+ // On Windows convert UTF16 encoded variable to UTF8
+ auto Len = wcslen(Var);
+ ArrayRef<char> Ref{reinterpret_cast<char const *>(Var),
+ Len * sizeof(*Var)};
+ EnvStorage.emplace_back();
+ auto convStatus = llvm::convertUTF16ToUTF8String(Ref, EnvStorage.back());
+ EXPECT_TRUE(convStatus);
+ return EnvStorage.back();
+#else
+ (void)this;
+ return StringRef(Var);
+#endif
+ };
+
+ while (*EnvP != nullptr) {
+ auto S = prepareEnvVar(*EnvP);
+ if (!StringRef(S).starts_with("GTEST_"))
+ EnvTable.emplace_back(S);
+ ++EnvP;
+ }
+ }
+
+ void TearDown() override {
+ EnvTable.clear();
+ EnvStorage.clear();
+ }
+
+ void addEnvVar(StringRef Var) { EnvTable.emplace_back(Var); }
+
+ ArrayRef<StringRef> getEnviron() const { return EnvTable; }
+};
+
+#if LLVM_ENABLE_ONDISK_CAS
+
+static Error emptyConstructor(MappedFileRegionArena &) {
+ return Error::success();
+}
+
+TEST_F(CASProgramTest, MappedFileRegionArenaTest) {
+ auto TestAllocator = [](StringRef Path) {
+ std::optional<MappedFileRegionArena> Alloc;
+ ASSERT_THAT_ERROR(
+ MappedFileRegionArena::create(Path, /*Capacity=*/10 * 1024 * 1024,
+ /*HeaderOffset=*/0, emptyConstructor)
+ .moveInto(Alloc),
+ Succeeded());
+
+ std::vector<unsigned *> AllocatedPtr;
+ AllocatedPtr.resize(100);
+ DefaultThreadPool Threads;
+ for (unsigned I = 0; I < 100; ++I) {
+ Threads.async(
+ [&](unsigned Idx) {
+ // Allocate a buffer that is larger than needed so allocator hits
+ // additional pages for test coverage.
+ unsigned *P = (unsigned *)cantFail(Alloc->allocate(100));
+ *P = Idx;
+ AllocatedPtr[Idx] = P;
+ },
+ I);
+ }
+
+ Threads.wait();
+ for (unsigned I = 0; I < 100; ++I)
+ EXPECT_EQ(*AllocatedPtr[I], I);
+ };
+
+ if (const char *File = getenv("LLVM_CAS_TEST_MAPPED_FILE_REGION")) {
+ TestAllocator(File);
+ exit(0);
+ }
+
+ SmallString<128> FilePath;
+ sys::fs::createUniqueDirectory("MappedFileRegionArena", FilePath);
+ sys::path::append(FilePath, "allocation-file");
+
+ std::string Executable =
+ sys::fs::getMainExecutable(TestMainArgv0, &ProgramID);
+ StringRef Argv[] = {
+ Executable, "--gtest_filter=CASProgramTest.MappedFileRegionArenaTest"};
+
+ // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
+ std::string EnvVar = "LLVM_CAS_TEST_MAPPED_FILE_REGION=";
+ EnvVar += FilePath.str();
+ addEnvVar(EnvVar);
+
+ std::string Error;
+ bool ExecutionFailed;
+ sys::ProcessInfo PI = sys::ExecuteNoWait(Executable, Argv, getEnviron(), {},
+ 0, &Error, &ExecutionFailed);
+ TestAllocator(FilePath);
+
+ ASSERT_FALSE(ExecutionFailed) << Error;
+ ASSERT_NE(PI.Pid, sys::ProcessInfo::InvalidPid) << "Invalid process id";
+ PI = llvm::sys::Wait(PI, /*SecondsToWait=*/5, &Error);
+ ASSERT_TRUE(PI.ReturnCode == 0);
+ ASSERT_TRUE(Error.empty());
+
+ // Clean up after both processes finish testing.
+ sys::fs::remove(FilePath);
+ sys::fs::remove_directories(sys::path::parent_path(FilePath));
+}
+
+TEST_F(CASProgramTest, MappedFileRegionArenaSizeTest) {
+ using namespace std::chrono_literals;
+ if (const char *File = getenv("LLVM_CAS_TEST_MAPPED_FILE_REGION")) {
+ ExponentialBackoff Backoff(5s);
+ do {
+ if (sys::fs::exists(File)) {
+ break;
+ }
+ } while (Backoff.waitForNextAttempt());
+
+ std::optional<MappedFileRegionArena> Alloc;
+ ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/1024,
+ /*HeaderOffset=*/0,
+ emptyConstructor)
+ .moveInto(Alloc),
+ Succeeded());
+ ASSERT_TRUE(Alloc->capacity() == 2048);
+
+ Alloc.reset();
+ ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/4096,
+ /*HeaderOffset=*/0,
+ emptyConstructor)
+ .moveInto(Alloc),
+ Succeeded());
+ ASSERT_TRUE(Alloc->capacity() == 2048);
+ Alloc.reset();
+
+ ASSERT_THAT_ERROR(
+ MappedFileRegionArena::create(File, /*Capacity=*/2048,
+ /*HeaderOffset=*/32, emptyConstructor)
+ .moveInto(Alloc),
+ FailedWithMessage(
+ "specified header offset (32) does not match existing config (0)"));
+
+ ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/2048,
+ /*HeaderOffset=*/0,
+ emptyConstructor)
+ .moveInto(Alloc),
+ Succeeded());
+
+ exit(0);
+ }
+
+ SmallString<128> FilePath;
+ sys::fs::createUniqueDirectory("MappedFileRegionArena", FilePath);
+ sys::path::append(FilePath, "allocation-file");
+
+ std::string Executable =
+ sys::fs::getMainExecutable(TestMainArgv0, &ProgramID);
+ StringRef Argv[] = {
+ Executable,
+ "--gtest_filter=CASProgramTest.MappedFileRegionArenaSizeTest"};
+
+ // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
+ std::string EnvVar = "LLVM_CAS_TEST_MAPPED_FILE_REGION=";
+ EnvVar += FilePath.str();
+ addEnvVar(EnvVar);
+
+ std::optional<MappedFileRegionArena> Alloc;
+ ASSERT_THAT_ERROR(MappedFileRegionArena::create(FilePath, /*Capacity=*/2048,
+ /*HeaderOffset=*/0,
+ emptyConstructor)
+ .moveInto(Alloc),
+ Succeeded());
+
+ std::string Error;
+ bool ExecutionFailed;
+ sys::ProcessInfo PI = sys::ExecuteNoWait(Executable, Argv, getEnviron(), {},
+ 0, &Error, &ExecutionFailed);
+
+ ASSERT_FALSE(ExecutionFailed) << Error;
+ ASSERT_NE(PI.Pid, sys::ProcessInfo::InvalidPid) << "Invalid process id";
+ PI = llvm::sys::Wait(PI, /*SecondsToWait=*/100, &Error);
+ ASSERT_TRUE(PI.ReturnCode == 0);
+ ASSERT_TRUE(Error.empty());
+
+ // Size is still the requested 2048.
+ ASSERT_TRUE(Alloc->capacity() == 2048);
+
+ // Clean up after both processes finish testing.
+ sys::fs::remove(FilePath);
+ sys::fs::remove_directories(sys::path::parent_path(FilePath));
+}
+
+#endif // LLVM_ENABLE_ONDISK_CAS
More information about the llvm-commits
mailing list