[llvm] Revert "[CAS] Add MappedFileRegionArena" (PR #158694)
Paul Kirth via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 15 10:32:20 PDT 2025
https://github.com/ilovepi created https://github.com/llvm/llvm-project/pull/158694
Reverts llvm/llvm-project#114099
This broke bots: https://lab.llvm.org/buildbot/#/builders/140/builds/30748
>From 7f74a55e6401f5c184acae1e5acbf56fe1402e9e Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth at google.com>
Date: Mon, 15 Sep 2025 10:31:12 -0700
Subject: [PATCH] Revert "[CAS] Add MappedFileRegionArena (#114099)"
This reverts commit f9cd2ee119ff6bd59f48fd71617dc4fc981083c9.
---
llvm/CMakeLists.txt | 1 -
llvm/include/llvm/CAS/MappedFileRegionArena.h | 130 ------
llvm/include/llvm/Config/llvm-config.h.cmake | 3 -
llvm/include/llvm/Support/FileSystem.h | 5 -
llvm/lib/CAS/CMakeLists.txt | 2 -
llvm/lib/CAS/MappedFileRegionArena.cpp | 388 ------------------
llvm/lib/CAS/OnDiskCommon.cpp | 121 ------
llvm/lib/CAS/OnDiskCommon.h | 46 ---
llvm/lib/Support/Unix/Path.inc | 5 -
llvm/lib/Support/Windows/Path.inc | 17 -
llvm/unittests/CAS/CMakeLists.txt | 5 -
llvm/unittests/CAS/ProgramTest.cpp | 239 -----------
12 files changed, 962 deletions(-)
delete mode 100644 llvm/include/llvm/CAS/MappedFileRegionArena.h
delete mode 100644 llvm/lib/CAS/MappedFileRegionArena.cpp
delete mode 100644 llvm/lib/CAS/OnDiskCommon.cpp
delete mode 100644 llvm/lib/CAS/OnDiskCommon.h
delete mode 100644 llvm/unittests/CAS/ProgramTest.cpp
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index e8af7fb432f40..b98192968a3ab 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -878,7 +878,6 @@ option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF)
option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON)
option (LLVM_ENABLE_BINDINGS "Build bindings." ON)
option (LLVM_ENABLE_TELEMETRY "Enable the telemetry library. If set to OFF, library cannot be enabled after build (eg., at runtime)" ON)
-option (LLVM_ENABLE_ONDISK_CAS "Build OnDiskCAS." ON)
set(LLVM_INSTALL_DOXYGEN_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/llvm/doxygen-html"
CACHE STRING "Doxygen-generated HTML documentation install directory")
diff --git a/llvm/include/llvm/CAS/MappedFileRegionArena.h b/llvm/include/llvm/CAS/MappedFileRegionArena.h
deleted file mode 100644
index ff51f0eb59929..0000000000000
--- a/llvm/include/llvm/CAS/MappedFileRegionArena.h
+++ /dev/null
@@ -1,130 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// This file declares interface for MappedFileRegionArena, a bump pointer
-/// allocator, backed by a memory-mapped file.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CAS_MAPPEDFILEREGIONARENA_H
-#define LLVM_CAS_MAPPEDFILEREGIONARENA_H
-
-#include "llvm/Support/Alignment.h"
-#include "llvm/Support/FileSystem.h"
-#include <atomic>
-
-namespace llvm::cas {
-
-/// Allocator for an owned mapped file region that supports thread-safe and
-/// process-safe bump pointer allocation.
-///
-/// This allocator is designed to create a sparse file when supported by the
-/// filesystem's \c ftruncate so that it can be used with a large maximum size.
-/// It will also attempt to shrink the underlying file down to its current
-/// allocation size when the last concurrent mapping is closed.
-///
-/// Process-safe. Uses file locks when resizing the file during initialization
-/// and destruction.
-///
-/// Thread-safe. Requires OS support thread-safe file lock.
-///
-/// Provides 8-byte alignment for all allocations.
-class MappedFileRegionArena {
-public:
- using RegionT = sys::fs::mapped_file_region;
-
- /// Header for MappedFileRegionArena. It can be configured to be located
- /// at any location within the file and the allocation will be appended after
- /// the header.
- struct Header {
- // BumpPtr for new allocation.
- std::atomic<uint64_t> BumpPtr;
- // Allocated size on disk.
- std::atomic<uint64_t> AllocatedSize;
- // Capacity of the file.
- std::atomic<uint64_t> Capacity;
- // Offset from the beginning of the file to this header (for verification).
- std::atomic<uint64_t> HeaderOffset;
- };
-
- /// Create a \c MappedFileRegionArena.
- ///
- /// \param Path the path to open the mapped region.
- /// \param Capacity the maximum size for the mapped file region.
- /// \param HeaderOffset the offset at which to store the header. This is so
- /// that information can be stored before the header, like a file magic.
- /// \param NewFileConstructor is for constructing new files. It has exclusive
- /// access to the file. Must call \c initializeBumpPtr.
- static Expected<MappedFileRegionArena>
- create(const Twine &Path, uint64_t Capacity, uint64_t HeaderOffset,
- function_ref<Error(MappedFileRegionArena &)> NewFileConstructor);
-
- /// Minimum alignment for allocations, currently hardcoded to 8B.
- static constexpr Align getAlign() {
- // Trick Align into giving us '8' as a constexpr.
- struct alignas(8) T {};
- static_assert(alignof(T) == 8, "Tautology failed?");
- return Align::Of<T>();
- }
-
- /// Allocate at least \p AllocSize. Rounds up to \a getAlign().
- Expected<char *> allocate(uint64_t AllocSize) {
- auto Offset = allocateOffset(AllocSize);
- if (LLVM_UNLIKELY(!Offset))
- return Offset.takeError();
- return data() + *Offset;
- }
- /// Allocate, returning the offset from \a data() instead of a pointer.
- Expected<int64_t> allocateOffset(uint64_t AllocSize);
-
- char *data() const { return Region.data(); }
- uint64_t size() const { return H->BumpPtr; }
- uint64_t capacity() const { return Region.size(); }
-
- RegionT &getRegion() { return Region; }
-
- ~MappedFileRegionArena() { destroyImpl(); }
-
- MappedFileRegionArena() = default;
- MappedFileRegionArena(MappedFileRegionArena &&RHS) { moveImpl(RHS); }
- MappedFileRegionArena &operator=(MappedFileRegionArena &&RHS) {
- destroyImpl();
- moveImpl(RHS);
- return *this;
- }
-
- MappedFileRegionArena(const MappedFileRegionArena &) = delete;
- MappedFileRegionArena &operator=(const MappedFileRegionArena &) = delete;
-
-private:
- // initialize header from offset.
- void initializeHeader(uint64_t HeaderOffset);
-
- void destroyImpl();
- void moveImpl(MappedFileRegionArena &RHS) {
- std::swap(Region, RHS.Region);
- std::swap(H, RHS.H);
- std::swap(Path, RHS.Path);
- std::swap(FD, RHS.FD);
- std::swap(SharedLockFD, RHS.SharedLockFD);
- }
-
-private:
- RegionT Region;
- Header *H = nullptr;
- std::string Path;
- // File descriptor for the main storage file.
- std::optional<int> FD;
- // File descriptor for the file used as reader/writer lock.
- std::optional<int> SharedLockFD;
-};
-
-} // namespace llvm::cas
-
-#endif // LLVM_CAS_MAPPEDFILEREGIONARENA_H
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 6488d6c01b5c6..39136bc45c292 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -146,7 +146,4 @@
coverage bugs, and to 0 otherwise. */
#cmakedefine01 LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
-/* Define to 1 to enable LLVM OnDisk Content Addressable Storage */
-#cmakedefine01 LLVM_ENABLE_ONDISK_CAS
-
#endif
diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index c203779307840..a21b0a272d2b0 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -410,11 +410,6 @@ LLVM_ABI std::error_code copy_file(const Twine &From, int ToFD);
/// platform-specific error_code.
LLVM_ABI std::error_code resize_file(int FD, uint64_t Size);
-/// Resize path to size with sparse files explicitly enabled. It uses
-/// FSCTL_SET_SPARSE On Windows. This is the same as resize_file on
-/// non-Windows
-LLVM_ABI std::error_code resize_file_sparse(int FD, uint64_t Size);
-
/// Resize \p FD to \p Size before mapping \a mapped_file_region::readwrite. On
/// non-Windows, this calls \a resize_file(). On Windows, this is a no-op,
/// since the subsequent mapping (via \c CreateFileMapping) automatically
diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt
index 6ed724bc2fd76..f3d2b41c704bc 100644
--- a/llvm/lib/CAS/CMakeLists.txt
+++ b/llvm/lib/CAS/CMakeLists.txt
@@ -3,9 +3,7 @@ add_llvm_component_library(LLVMCAS
ActionCaches.cpp
BuiltinCAS.cpp
InMemoryCAS.cpp
- MappedFileRegionArena.cpp
ObjectStore.cpp
- OnDiskCommon.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/CAS
diff --git a/llvm/lib/CAS/MappedFileRegionArena.cpp b/llvm/lib/CAS/MappedFileRegionArena.cpp
deleted file mode 100644
index 3c920edcaae6a..0000000000000
--- a/llvm/lib/CAS/MappedFileRegionArena.cpp
+++ /dev/null
@@ -1,388 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-/// \file Implements MappedFileRegionArena.
-///
-/// A bump pointer allocator, backed by a memory-mapped file.
-///
-/// The effect we want is:
-///
-/// Step 1. If it doesn't exist, create the file with an initial size.
-/// Step 2. Reserve virtual memory large enough for the max file size.
-/// Step 3. Map the file into memory in the reserved region.
-/// Step 4. Increase the file size and update the mapping when necessary.
-///
-/// However, updating the mapping is challenging when it needs to work portably,
-/// and across multiple processes without locking for every read. Our current
-/// implementation handles the steps above in following ways:
-///
-/// Step 1. Use \ref sys::fs::resize_file_sparse to grow the file to its max
-/// size (typically several GB). If the file system doesn't support
-/// sparse file, this may return a fully allocated file.
-/// Step 2. Call \ref sys::fs::mapped_file_region to map the entire file.
-/// Step 3. [Automatic as part of step 2.]
-/// Step 4. If supported, use \c fallocate or similiar APIs to ensure the file
-/// system storage for the sparse file so we won't end up with partial
-/// file if the disk is out of space.
-///
-/// Additionally, we attempt to resize the file to its actual data size when
-/// closing the mapping, if this is the only concurrent instance. This is done
-/// using file locks. Shrinking the file mitigates problems with having large
-/// files: on filesystems without sparse files it avoids unnecessary space use;
-/// it also avoids allocating the full size if another process copies the file,
-/// which typically loses sparseness. These mitigations only work while the file
-/// is not in use.
-///
-/// The capacity and the header offset is determined by the first user of the
-/// MappedFileRegionArena instance and any future mismatched value from the
-/// original will result in error on creation.
-///
-/// To support resizing, we use two separate file locks:
-/// 1. We use a shared reader lock on a ".shared" file until destruction.
-/// 2. We use a lock on the main file during initialization - shared to check
-/// the status, upgraded to exclusive to resize/initialize the file.
-///
-/// Then during destruction we attempt to get exclusive access on (1), which
-/// requires no concurrent readers. If so, we shrink the file. Using two
-/// separate locks simplifies the implementation and enables it to work on
-/// platforms (e.g. Windows) where a shared/reader lock prevents writing.
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CAS/MappedFileRegionArena.h"
-#include "OnDiskCommon.h"
-#include "llvm/ADT/StringExtras.h"
-
-#if LLVM_ON_UNIX
-#include <sys/stat.h>
-#if __has_include(<sys/param.h>)
-#include <sys/param.h>
-#endif
-#ifdef DEV_BSIZE
-#define MAPPED_FILE_BSIZE DEV_BSIZE
-#elif __linux__
-#define MAPPED_FILE_BSIZE 512
-#endif
-#endif
-
-using namespace llvm;
-using namespace llvm::cas;
-using namespace llvm::cas::ondisk;
-
-namespace {
-struct FileWithLock {
- std::string Path;
- int FD = -1;
- std::optional<sys::fs::LockKind> Locked;
-
-private:
- FileWithLock(std::string PathStr, Error &E) : Path(std::move(PathStr)) {
- ErrorAsOutParameter EOP(&E);
- if (std::error_code EC = sys::fs::openFileForReadWrite(
- Path, FD, sys::fs::CD_OpenAlways, sys::fs::OF_None))
- E = createFileError(Path, EC);
- }
-
-public:
- FileWithLock(FileWithLock &) = delete;
- FileWithLock(FileWithLock &&Other) {
- Path = std::move(Other.Path);
- FD = Other.FD;
- Other.FD = -1;
- Locked = Other.Locked;
- Other.Locked = std::nullopt;
- }
-
- ~FileWithLock() { consumeError(unlock()); }
-
- static Expected<FileWithLock> open(StringRef Path) {
- Error E = Error::success();
- FileWithLock Result(Path.str(), E);
- if (E)
- return std::move(E);
- return std::move(Result);
- }
-
- Error lock(sys::fs::LockKind LK) {
- assert(!Locked && "already locked");
- if (std::error_code EC = lockFileThreadSafe(FD, LK))
- return createFileError(Path, EC);
- Locked = LK;
- return Error::success();
- }
-
- Error switchLock(sys::fs::LockKind LK) {
- assert(Locked && "not locked");
- if (auto E = unlock())
- return E;
-
- return lock(LK);
- }
-
- Error unlock() {
- if (Locked) {
- Locked = std::nullopt;
- if (std::error_code EC = unlockFileThreadSafe(FD))
- return createFileError(Path, EC);
- }
- return Error::success();
- }
-
- // Return true if succeed to lock the file exclusively.
- bool tryLockExclusive() {
- assert(!Locked && "can only try to lock if not locked");
- if (tryLockFileThreadSafe(FD) == std::error_code()) {
- Locked = sys::fs::LockKind::Exclusive;
- return true;
- }
-
- return false;
- }
-
- // Release the lock so it will not be unlocked on destruction.
- void release() {
- Locked = std::nullopt;
- FD = -1;
- }
-};
-
-struct FileSizeInfo {
- uint64_t Size;
- uint64_t AllocatedSize;
-
- static ErrorOr<FileSizeInfo> get(sys::fs::file_t File);
-};
-} // end anonymous namespace
-
-Expected<MappedFileRegionArena> MappedFileRegionArena::create(
- const Twine &Path, uint64_t Capacity, uint64_t HeaderOffset,
- function_ref<Error(MappedFileRegionArena &)> NewFileConstructor) {
- uint64_t MinCapacity = HeaderOffset + sizeof(Header);
- if (Capacity < MinCapacity)
- return createStringError(
- std::make_error_code(std::errc::invalid_argument),
- "capacity is too small to hold MappedFileRegionArena");
-
- MappedFileRegionArena Result;
- Result.Path = Path.str();
-
- // Open the shared lock file. See file comment for details of locking scheme.
- SmallString<128> SharedFilePath(Result.Path);
- SharedFilePath.append(".shared");
-
- auto SharedFileLock = FileWithLock::open(SharedFilePath);
- if (!SharedFileLock)
- return SharedFileLock.takeError();
- Result.SharedLockFD = SharedFileLock->FD;
-
- // Take shared/reader lock that will be held until destroyImpl if construction
- // is successful.
- if (auto E = SharedFileLock->lock(sys::fs::LockKind::Shared))
- return std::move(E);
-
- // Take shared/reader lock for initialization.
- auto MainFile = FileWithLock::open(Result.Path);
- if (!MainFile)
- return MainFile.takeError();
- if (Error E = MainFile->lock(sys::fs::LockKind::Shared))
- return std::move(E);
- Result.FD = MainFile->FD;
-
- sys::fs::file_t File = sys::fs::convertFDToNativeFile(MainFile->FD);
- auto FileSize = FileSizeInfo::get(File);
- if (!FileSize)
- return createFileError(Result.Path, FileSize.getError());
-
- // If the size is smaller than the capacity, we need to initialize the file.
- // It maybe empty, or may have been shrunk during a previous close.
- if (FileSize->Size < Capacity) {
- // Lock the file exclusively so only one process will do the initialization.
- if (Error E = MainFile->switchLock(sys::fs::LockKind::Exclusive))
- return std::move(E);
- // Retrieve the current size now that we have exclusive access.
- FileSize = FileSizeInfo::get(File);
- if (!FileSize)
- return createFileError(Result.Path, FileSize.getError());
- }
-
- if (FileSize->Size >= MinCapacity) {
- // File is initialized. Read out the header to check for capacity and
- // offset.
- SmallVector<char, sizeof(Header)> HeaderContent(sizeof(Header));
- auto Size = sys::fs::readNativeFileSlice(File, HeaderContent, HeaderOffset);
- if (!Size)
- return Size.takeError();
-
- Header *H = reinterpret_cast<Header *>(HeaderContent.data());
- if (H->HeaderOffset != HeaderOffset)
- return createStringError(
- std::make_error_code(std::errc::invalid_argument),
- "specified header offset (" + utostr(HeaderOffset) +
- ") does not match existing config (" + utostr(H->HeaderOffset) +
- ")");
-
- // If the capacity doesn't match, use the existing capacity instead.
- if (H->Capacity != Capacity)
- Capacity = H->Capacity;
- }
-
- // If the size is smaller than capacity, we need to resize the file.
- if (FileSize->Size < Capacity) {
- assert(MainFile->Locked == sys::fs::LockKind::Exclusive);
- if (std::error_code EC =
- sys::fs::resize_file_sparse(MainFile->FD, Capacity))
- return createFileError(Result.Path, EC);
- }
-
- // Create the mapped region.
- {
- std::error_code EC;
- sys::fs::mapped_file_region Map(
- File, sys::fs::mapped_file_region::readwrite, Capacity, 0, EC);
- if (EC)
- return createFileError(Result.Path, EC);
- Result.Region = std::move(Map);
- }
-
- // Initialize the header.
- Result.initializeHeader(HeaderOffset);
- if (FileSize->Size < MinCapacity) {
- assert(MainFile->Locked == sys::fs::LockKind::Exclusive);
- // If we need to fully initialize the file, call NewFileConstructor.
- if (Error E = NewFileConstructor(Result))
- return std::move(E);
-
- Result.H->HeaderOffset.exchange(HeaderOffset);
- Result.H->Capacity.exchange(Capacity);
- }
-
- if (MainFile->Locked == sys::fs::LockKind::Exclusive) {
- // If holding an exclusive lock, we might have resized the file and
- // performed some read/write to the file. Query the file size again to make
- // sure everything is up-to-date. Otherwise, FileSize info is already
- // up-to-date.
- FileSize = FileSizeInfo::get(File);
- if (!FileSize)
- return createFileError(Result.Path, FileSize.getError());
- Result.H->AllocatedSize.exchange(FileSize->AllocatedSize);
- }
-
- // Release the shared lock so it can be closed in destoryImpl().
- SharedFileLock->release();
- return Result;
-}
-
-void MappedFileRegionArena::destroyImpl() {
- if (!FD)
- return;
-
- // Drop the shared lock indicating we are no longer accessing the file.
- if (SharedLockFD)
- (void)unlockFileThreadSafe(*SharedLockFD);
-
- // Attempt to truncate the file if we can get exclusive access. Ignore any
- // errors.
- if (H) {
- assert(SharedLockFD && "Must have shared lock file open");
- if (tryLockFileThreadSafe(*SharedLockFD) == std::error_code()) {
- size_t Size = size();
- // sync to file system to make sure all contents are up-to-date.
- (void)Region.sync();
- // unmap the file before resizing since that is the requirement for
- // some platforms.
- Region.unmap();
- (void)sys::fs::resize_file(*FD, Size);
- (void)unlockFileThreadSafe(*SharedLockFD);
- }
- }
-
- auto Close = [](std::optional<int> &FD) {
- if (FD) {
- sys::fs::file_t File = sys::fs::convertFDToNativeFile(*FD);
- sys::fs::closeFile(File);
- FD = std::nullopt;
- }
- };
-
- // Close the file and shared lock.
- Close(FD);
- Close(SharedLockFD);
-}
-
-void MappedFileRegionArena::initializeHeader(uint64_t HeaderOffset) {
- assert(capacity() < (uint64_t)INT64_MAX && "capacity must fit in int64_t");
- uint64_t HeaderEndOffset = HeaderOffset + sizeof(decltype(*H));
- assert(HeaderEndOffset <= capacity() &&
- "Expected end offset to be pre-allocated");
- assert(isAligned(Align::Of<decltype(*H)>(), HeaderOffset) &&
- "Expected end offset to be aligned");
- H = reinterpret_cast<decltype(H)>(data() + HeaderOffset);
-
- uint64_t ExistingValue = 0;
- if (!H->BumpPtr.compare_exchange_strong(ExistingValue, HeaderEndOffset))
- assert(ExistingValue >= HeaderEndOffset &&
- "Expected 0, or past the end of the header itself");
-}
-
-static Error createAllocatorOutOfSpaceError() {
- return createStringError(std::make_error_code(std::errc::not_enough_memory),
- "memory mapped file allocator is out of space");
-}
-
-Expected<int64_t> MappedFileRegionArena::allocateOffset(uint64_t AllocSize) {
- AllocSize = alignTo(AllocSize, getAlign());
- uint64_t OldEnd = H->BumpPtr.fetch_add(AllocSize);
- uint64_t NewEnd = OldEnd + AllocSize;
- if (LLVM_UNLIKELY(NewEnd > capacity())) {
- // Return the allocation. If the start already passed the end, that means
- // some other concurrent allocations already consumed all the capacity.
- // There is no need to return the original value. If the start was not
- // passed the end, current allocation certainly bumped it passed the end.
- // All other allocation afterwards must have failed and current allocation
- // is in charge of return the allocation back to a valid value.
- if (OldEnd <= capacity())
- (void)H->BumpPtr.exchange(OldEnd);
-
- return createAllocatorOutOfSpaceError();
- }
-
- uint64_t DiskSize = H->AllocatedSize;
- if (LLVM_UNLIKELY(NewEnd > DiskSize)) {
- uint64_t NewSize;
- // The minimum increment is a page, but allocate more to amortize the cost.
- constexpr uint64_t Increment = 1 * 1024 * 1024; // 1 MB
- if (Error E = preallocateFileTail(*FD, DiskSize, DiskSize + Increment)
- .moveInto(NewSize))
- return std::move(E);
- assert(NewSize >= DiskSize + Increment);
- // FIXME: on Darwin this can under-count the size if there is a race to
- // preallocate disk, because the semantics of F_PREALLOCATE are to add bytes
- // to the end of the file, not to allocate up to a fixed size.
- // Any discrepancy will be resolved the next time the file is truncated and
- // then reopend.
- while (DiskSize < NewSize)
- H->AllocatedSize.compare_exchange_strong(DiskSize, NewSize);
- }
- return OldEnd;
-}
-
-ErrorOr<FileSizeInfo> FileSizeInfo::get(sys::fs::file_t File) {
-#if LLVM_ON_UNIX && defined(MAPPED_FILE_BSIZE)
- struct stat Status;
- int StatRet = ::fstat(File, &Status);
- if (StatRet)
- return errnoAsErrorCode();
- uint64_t AllocatedSize = uint64_t(Status.st_blksize) * MAPPED_FILE_BSIZE;
- return FileSizeInfo{uint64_t(Status.st_size), AllocatedSize};
-#else
- // Fallback: assume the file is fully allocated. Note: this may result in
- // data loss on out-of-space.
- sys::fs::file_status Status;
- if (std::error_code EC = sys::fs::status(File, Status))
- return EC;
- return FileSizeInfo{Status.getSize(), Status.getSize()};
-#endif
-}
diff --git a/llvm/lib/CAS/OnDiskCommon.cpp b/llvm/lib/CAS/OnDiskCommon.cpp
deleted file mode 100644
index f7765c58944c5..0000000000000
--- a/llvm/lib/CAS/OnDiskCommon.cpp
+++ /dev/null
@@ -1,121 +0,0 @@
-//===- OnDiskCommon.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "OnDiskCommon.h"
-#include "llvm/Config/config.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
-#include <thread>
-
-#if __has_include(<sys/file.h>)
-#include <sys/file.h>
-#ifdef LOCK_SH
-#define HAVE_FLOCK 1
-#else
-#define HAVE_FLOCK 0
-#endif
-#endif
-
-#if __has_include(<fcntl.h>)
-#include <fcntl.h>
-#endif
-
-using namespace llvm;
-
-std::error_code cas::ondisk::lockFileThreadSafe(int FD,
- sys::fs::LockKind Kind) {
-#if HAVE_FLOCK
- if (flock(FD, Kind == sys::fs::LockKind::Exclusive ? LOCK_EX : LOCK_SH) == 0)
- return std::error_code();
- return std::error_code(errno, std::generic_category());
-#elif defined(_WIN32)
- // On Windows this implementation is thread-safe.
- return sys::fs::lockFile(FD, Kind);
-#else
- return make_error_code(std::errc::no_lock_available);
-#endif
-}
-
-std::error_code cas::ondisk::unlockFileThreadSafe(int FD) {
-#if HAVE_FLOCK
- if (flock(FD, LOCK_UN) == 0)
- return std::error_code();
- return std::error_code(errno, std::generic_category());
-#elif defined(_WIN32)
- // On Windows this implementation is thread-safe.
- return sys::fs::unlockFile(FD);
-#else
- return make_error_code(std::errc::no_lock_available);
-#endif
-}
-
-std::error_code
-cas::ondisk::tryLockFileThreadSafe(int FD, std::chrono::milliseconds Timeout,
- sys::fs::LockKind Kind) {
-#if HAVE_FLOCK
- auto Start = std::chrono::steady_clock::now();
- auto End = Start + Timeout;
- do {
- if (flock(FD, (Kind == sys::fs::LockKind::Exclusive ? LOCK_EX : LOCK_SH) |
- LOCK_NB) == 0)
- return std::error_code();
- int Error = errno;
- if (Error == EWOULDBLOCK) {
- // Match sys::fs::tryLockFile, which sleeps for 1 ms per attempt.
- std::this_thread::sleep_for(std::chrono::milliseconds(1));
- continue;
- }
- return std::error_code(Error, std::generic_category());
- } while (std::chrono::steady_clock::now() < End);
- return make_error_code(std::errc::no_lock_available);
-#elif defined(_WIN32)
- // On Windows this implementation is thread-safe.
- return sys::fs::tryLockFile(FD, Timeout, Kind);
-#else
- return make_error_code(std::errc::no_lock_available);
-#endif
-}
-
-Expected<size_t> cas::ondisk::preallocateFileTail(int FD, size_t CurrentSize,
- size_t NewSize) {
- auto CreateError = [&](std::error_code EC) -> Expected<size_t> {
- if (EC == std::errc::not_supported)
- // Ignore ENOTSUP in case the filesystem cannot preallocate.
- return NewSize;
-#if defined(HAVE_POSIX_FALLOCATE)
- if (EC == std::errc::invalid_argument && CurrentSize < NewSize && // len > 0
- NewSize < std::numeric_limits<off_t>::max()) // 0 <= offset, len < max
- // Prior to 2024, POSIX required EINVAL for cases that should be ENOTSUP,
- // so handle it the same as above if it is not one of the other ways to
- // get EINVAL.
- return NewSize;
-#endif
- return createStringError(EC,
- "failed to allocate to CAS file: " + EC.message());
- };
-#if defined(HAVE_POSIX_FALLOCATE)
- // Note: posix_fallocate returns its error directly, not via errno.
- if (int Err = posix_fallocate(FD, CurrentSize, NewSize - CurrentSize))
- return CreateError(std::error_code(Err, std::generic_category()));
- return NewSize;
-#elif defined(__APPLE__)
- fstore_t FAlloc;
- FAlloc.fst_flags = F_ALLOCATEALL | F_ALLOCATEPERSIST;
- FAlloc.fst_posmode = F_PEOFPOSMODE;
- FAlloc.fst_offset = 0;
- FAlloc.fst_length = NewSize - CurrentSize;
- FAlloc.fst_bytesalloc = 0;
- if (fcntl(FD, F_PREALLOCATE, &FAlloc))
- return CreateError(errnoAsErrorCode());
- assert(CurrentSize + FAlloc.fst_bytesalloc >= NewSize);
- return CurrentSize + FAlloc.fst_bytesalloc;
-#else
- (void)CreateError; // Silence unused variable.
- return NewSize; // Pretend it worked.
-#endif
-}
diff --git a/llvm/lib/CAS/OnDiskCommon.h b/llvm/lib/CAS/OnDiskCommon.h
deleted file mode 100644
index 8b79ffe5c3158..0000000000000
--- a/llvm/lib/CAS/OnDiskCommon.h
+++ /dev/null
@@ -1,46 +0,0 @@
-//===- OnDiskCommon.h -------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_CAS_ONDISKCOMMON_H
-#define LLVM_LIB_CAS_ONDISKCOMMON_H
-
-#include "llvm/Support/Error.h"
-#include "llvm/Support/FileSystem.h"
-#include <chrono>
-
-namespace llvm::cas::ondisk {
-
-/// Thread-safe alternative to \c sys::fs::lockFile. This does not support all
-/// the platforms that \c sys::fs::lockFile does, so keep it in the CAS library
-/// for now.
-std::error_code lockFileThreadSafe(int FD, llvm::sys::fs::LockKind Kind);
-
-/// Thread-safe alternative to \c sys::fs::unlockFile. This does not support all
-/// the platforms that \c sys::fs::lockFile does, so keep it in the CAS library
-/// for now.
-std::error_code unlockFileThreadSafe(int FD);
-
-/// Thread-safe alternative to \c sys::fs::tryLockFile. This does not support
-/// all the platforms that \c sys::fs::lockFile does, so keep it in the CAS
-/// library for now.
-std::error_code tryLockFileThreadSafe(
- int FD, std::chrono::milliseconds Timeout = std::chrono::milliseconds(0),
- llvm::sys::fs::LockKind Kind = llvm::sys::fs::LockKind::Exclusive);
-
-/// Allocate space for the file \p FD on disk, if the filesystem supports it.
-///
-/// On filesystems that support this operation, this ensures errors such as
-/// \c std::errc::no_space_on_device are detected before we write data.
-///
-/// \returns the new size of the file, or an \c Error.
-Expected<size_t> preallocateFileTail(int FD, size_t CurrentSize,
- size_t NewSize);
-
-} // namespace llvm::cas::ondisk
-
-#endif // LLVM_LIB_CAS_ONDISKCOMMON_H
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index 0d991ead72416..2f563e2899b56 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -600,11 +600,6 @@ std::error_code resize_file(int FD, uint64_t Size) {
return std::error_code();
}
-std::error_code resize_file_sparse(int FD, uint64_t Size) {
- // On Unix, this is the same as `resize_file`.
- return resize_file(FD, Size);
-}
-
static int convertAccessMode(AccessMode Mode) {
switch (Mode) {
case AccessMode::Exist:
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index be007b7abdb51..6672d8e0ec777 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -27,7 +27,6 @@
#include "llvm/Support/Windows/WindowsSupport.h"
#include <shellapi.h>
#include <shlobj.h>
-#include <winioctl.h>
#undef max
@@ -618,22 +617,6 @@ std::error_code resize_file(int FD, uint64_t Size) {
return std::error_code(error, std::generic_category());
}
-std::error_code resize_file_sparse(int FD, uint64_t Size) {
- HANDLE hFile = reinterpret_cast<HANDLE>(::_get_osfhandle(FD));
- DWORD temp;
- if (!DeviceIoControl(hFile, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &temp,
- NULL)) {
- return mapWindowsError(GetLastError());
- }
- LARGE_INTEGER liSize;
- liSize.QuadPart = Size;
- if (!SetFilePointerEx(hFile, liSize, NULL, FILE_BEGIN) ||
- !SetEndOfFile(hFile)) {
- return mapWindowsError(GetLastError());
- }
- return std::error_code();
-}
-
std::error_code access(const Twine &Path, AccessMode Mode) {
SmallVector<wchar_t, 128> PathUtf16;
diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt
index ab709e30369bf..ff081007f31bc 100644
--- a/llvm/unittests/CAS/CMakeLists.txt
+++ b/llvm/unittests/CAS/CMakeLists.txt
@@ -1,7 +1,3 @@
-if (LLVM_ENABLE_ONDISK_CAS)
- add_definitions(-DLLVM_ENABLE_ONDISK_CAS=1)
-endif()
-
set(LLVM_LINK_COMPONENTS
Support
CAS
@@ -12,7 +8,6 @@ add_llvm_unittest(CASTests
ActionCacheTest.cpp
CASTestConfig.cpp
ObjectStoreTest.cpp
- ProgramTest.cpp
)
target_link_libraries(CASTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/CAS/ProgramTest.cpp b/llvm/unittests/CAS/ProgramTest.cpp
deleted file mode 100644
index 3ba5f697eeed5..0000000000000
--- a/llvm/unittests/CAS/ProgramTest.cpp
+++ /dev/null
@@ -1,239 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/Program.h"
-#include "llvm/CAS/MappedFileRegionArena.h"
-#include "llvm/Config/llvm-config.h"
-#include "llvm/Support/ConvertUTF.h"
-#include "llvm/Support/ExponentialBackoff.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/ThreadPool.h"
-#include "llvm/Testing/Support/Error.h"
-#include "gtest/gtest.h"
-#if defined(__APPLE__)
-#include <crt_externs.h>
-#elif !defined(_MSC_VER)
-// Forward declare environ in case it's not provided by stdlib.h.
-extern char **environ;
-#endif
-
-using namespace llvm;
-using namespace llvm::cas;
-
-extern const char *TestMainArgv0;
-static char ProgramID = 0;
-
-class CASProgramTest : public testing::Test {
- std::vector<StringRef> EnvTable;
- std::vector<std::string> EnvStorage;
-
-protected:
- void SetUp() override {
- auto EnvP = [] {
-#if defined(_WIN32)
- _wgetenv(L"TMP"); // Populate _wenviron, initially is null
- return _wenviron;
-#elif defined(__APPLE__)
- return *_NSGetEnviron();
-#else
- return environ;
-#endif
- }();
- ASSERT_TRUE(EnvP);
-
- auto prepareEnvVar = [this](decltype(*EnvP) Var) -> StringRef {
-#if defined(_WIN32)
- // On Windows convert UTF16 encoded variable to UTF8
- auto Len = wcslen(Var);
- ArrayRef<char> Ref{reinterpret_cast<char const *>(Var),
- Len * sizeof(*Var)};
- EnvStorage.emplace_back();
- auto convStatus = llvm::convertUTF16ToUTF8String(Ref, EnvStorage.back());
- EXPECT_TRUE(convStatus);
- return EnvStorage.back();
-#else
- (void)this;
- return StringRef(Var);
-#endif
- };
-
- while (*EnvP != nullptr) {
- auto S = prepareEnvVar(*EnvP);
- if (!StringRef(S).starts_with("GTEST_"))
- EnvTable.emplace_back(S);
- ++EnvP;
- }
- }
-
- void TearDown() override {
- EnvTable.clear();
- EnvStorage.clear();
- }
-
- void addEnvVar(StringRef Var) { EnvTable.emplace_back(Var); }
-
- ArrayRef<StringRef> getEnviron() const { return EnvTable; }
-};
-
-#if LLVM_ENABLE_ONDISK_CAS
-
-static Error emptyConstructor(MappedFileRegionArena &) {
- return Error::success();
-}
-
-TEST_F(CASProgramTest, MappedFileRegionArenaTest) {
- auto TestAllocator = [](StringRef Path) {
- std::optional<MappedFileRegionArena> Alloc;
- ASSERT_THAT_ERROR(
- MappedFileRegionArena::create(Path, /*Capacity=*/10 * 1024 * 1024,
- /*HeaderOffset=*/0, emptyConstructor)
- .moveInto(Alloc),
- Succeeded());
-
- std::vector<unsigned *> AllocatedPtr;
- AllocatedPtr.resize(100);
- DefaultThreadPool Threads;
- for (unsigned I = 0; I < 100; ++I) {
- Threads.async(
- [&](unsigned Idx) {
- // Allocate a buffer that is larger than needed so allocator hits
- // additional pages for test coverage.
- unsigned *P = (unsigned *)cantFail(Alloc->allocate(100));
- *P = Idx;
- AllocatedPtr[Idx] = P;
- },
- I);
- }
-
- Threads.wait();
- for (unsigned I = 0; I < 100; ++I)
- EXPECT_EQ(*AllocatedPtr[I], I);
- };
-
- if (const char *File = getenv("LLVM_CAS_TEST_MAPPED_FILE_REGION")) {
- TestAllocator(File);
- exit(0);
- }
-
- SmallString<128> FilePath;
- sys::fs::createUniqueDirectory("MappedFileRegionArena", FilePath);
- sys::path::append(FilePath, "allocation-file");
-
- std::string Executable =
- sys::fs::getMainExecutable(TestMainArgv0, &ProgramID);
- StringRef Argv[] = {
- Executable, "--gtest_filter=CASProgramTest.MappedFileRegionArenaTest"};
-
- // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
- std::string EnvVar = "LLVM_CAS_TEST_MAPPED_FILE_REGION=";
- EnvVar += FilePath.str();
- addEnvVar(EnvVar);
-
- std::string Error;
- bool ExecutionFailed;
- sys::ProcessInfo PI = sys::ExecuteNoWait(Executable, Argv, getEnviron(), {},
- 0, &Error, &ExecutionFailed);
- TestAllocator(FilePath);
-
- ASSERT_FALSE(ExecutionFailed) << Error;
- ASSERT_NE(PI.Pid, sys::ProcessInfo::InvalidPid) << "Invalid process id";
- PI = llvm::sys::Wait(PI, /*SecondsToWait=*/5, &Error);
- ASSERT_TRUE(PI.ReturnCode == 0);
- ASSERT_TRUE(Error.empty());
-
- // Clean up after both processes finish testing.
- sys::fs::remove(FilePath);
- sys::fs::remove_directories(sys::path::parent_path(FilePath));
-}
-
-TEST_F(CASProgramTest, MappedFileRegionArenaSizeTest) {
- using namespace std::chrono_literals;
- if (const char *File = getenv("LLVM_CAS_TEST_MAPPED_FILE_REGION")) {
- ExponentialBackoff Backoff(5s);
- do {
- if (sys::fs::exists(File)) {
- break;
- }
- } while (Backoff.waitForNextAttempt());
-
- std::optional<MappedFileRegionArena> Alloc;
- ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/1024,
- /*HeaderOffset=*/0,
- emptyConstructor)
- .moveInto(Alloc),
- Succeeded());
- ASSERT_TRUE(Alloc->capacity() == 2048);
-
- Alloc.reset();
- ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/4096,
- /*HeaderOffset=*/0,
- emptyConstructor)
- .moveInto(Alloc),
- Succeeded());
- ASSERT_TRUE(Alloc->capacity() == 2048);
- Alloc.reset();
-
- ASSERT_THAT_ERROR(
- MappedFileRegionArena::create(File, /*Capacity=*/2048,
- /*HeaderOffset=*/32, emptyConstructor)
- .moveInto(Alloc),
- FailedWithMessage(
- "specified header offset (32) does not match existing config (0)"));
-
- ASSERT_THAT_ERROR(MappedFileRegionArena::create(File, /*Capacity=*/2048,
- /*HeaderOffset=*/0,
- emptyConstructor)
- .moveInto(Alloc),
- Succeeded());
-
- exit(0);
- }
-
- SmallString<128> FilePath;
- sys::fs::createUniqueDirectory("MappedFileRegionArena", FilePath);
- sys::path::append(FilePath, "allocation-file");
-
- std::string Executable =
- sys::fs::getMainExecutable(TestMainArgv0, &ProgramID);
- StringRef Argv[] = {
- Executable,
- "--gtest_filter=CASProgramTest.MappedFileRegionArenaSizeTest"};
-
- // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
- std::string EnvVar = "LLVM_CAS_TEST_MAPPED_FILE_REGION=";
- EnvVar += FilePath.str();
- addEnvVar(EnvVar);
-
- std::optional<MappedFileRegionArena> Alloc;
- ASSERT_THAT_ERROR(MappedFileRegionArena::create(FilePath, /*Capacity=*/2048,
- /*HeaderOffset=*/0,
- emptyConstructor)
- .moveInto(Alloc),
- Succeeded());
-
- std::string Error;
- bool ExecutionFailed;
- sys::ProcessInfo PI = sys::ExecuteNoWait(Executable, Argv, getEnviron(), {},
- 0, &Error, &ExecutionFailed);
-
- ASSERT_FALSE(ExecutionFailed) << Error;
- ASSERT_NE(PI.Pid, sys::ProcessInfo::InvalidPid) << "Invalid process id";
- PI = llvm::sys::Wait(PI, /*SecondsToWait=*/100, &Error);
- ASSERT_TRUE(PI.ReturnCode == 0);
- ASSERT_TRUE(Error.empty());
-
- // Size is still the requested 2048.
- ASSERT_TRUE(Alloc->capacity() == 2048);
-
- // Clean up after both processes finish testing.
- sys::fs::remove(FilePath);
- sys::fs::remove_directories(sys::path::parent_path(FilePath));
-}
-
-#endif // LLVM_ENABLE_ONDISK_CAS
More information about the llvm-commits
mailing list