[llvm] [CAS] Add ObjectStore::getStandaloneMemoryBuffer() (PR #215360)
Steven Wu via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 13:24:08 PDT 2026
https://github.com/cachemeifyoucan updated https://github.com/llvm/llvm-project/pull/215360
>From 1b89acafe2a504c14b8f3a7ba311aec371e386a8 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Mon, 10 Aug 2026 11:31:30 -0700
Subject: [PATCH 1/4] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
llvm/docs/ContentAddressableStorage.md | 10 ++
llvm/docs/ReleaseNotes.md | 6 +
llvm/include/llvm/CAS/ObjectStore.h | 34 +++++-
llvm/include/llvm/CAS/OnDiskGraphDB.h | 14 +++
llvm/lib/CAS/ObjectStore.cpp | 19 ++++
llvm/lib/CAS/OnDiskCAS.cpp | 12 ++
llvm/lib/CAS/OnDiskGraphDB.cpp | 78 +++++++++++++
llvm/unittests/CAS/ObjectStoreTest.cpp | 148 +++++++++++++++++++++++++
8 files changed, 317 insertions(+), 4 deletions(-)
diff --git a/llvm/docs/ContentAddressableStorage.md b/llvm/docs/ContentAddressableStorage.md
index a42cbe731bf25..ea8c6a2d2e39b 100644
--- a/llvm/docs/ContentAddressableStorage.md
+++ b/llvm/docs/ContentAddressableStorage.md
@@ -61,6 +61,12 @@ of error handling. The class APIs also provide convenient methods to
access underlying data. The lifetime of the underlying data is equal to
the lifetime of the instance of `ObjectStore` unless explicitly copied.
+To get a lifetime-extended buffer, call `getStandaloneMemoryBuffer()`, which
+returns a `MemoryBuffer` that remains valid after the `ObjectStore` is
+destroyed. A CAS can provide a customized implementation that is cheaper than
+a copy; the on-disk CAS maps the object's file where it has one, so the pages
+stay evictable instead of being charged as dirty memory.
+
### CASID
`CASID` is the hash identifier for CASObjects. It owns the underlying
@@ -106,6 +112,10 @@ ones.
To add your own implementation, you just need to add a subclass to
`llvm::cas::ObjectStore` and implement all its pure virtual methods.
+`getStandaloneMemoryBufferImpl()` is an optional customization point; it
+copies by default, so override it only if the implementation can hand out
+storage that outlives itself.
+
To be interchangeable with LLVM ObjectStore, the new CAS implementation
needs to conform to following contracts:
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index efd33a907cfa7..dc381334f62b8 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -54,6 +54,12 @@ Makes programs 10x faster by doing Special New Thing.
### Changes to LLVM infrastructure
+* `cas::ObjectStore::getMemoryBuffer()` was documented as returning a buffer
+ whose lifetime is independent of the CAS, but the buffer it returns may alias
+ storage the CAS owns and so cannot outlive it. The documentation now matches
+ the behavior, and the new `getStandaloneMemoryBuffer()` provides a buffer that
+ does stay valid after the `ObjectStore` is destroyed.
+
### Changes to building LLVM
### Changes to TableGen
diff --git a/llvm/include/llvm/CAS/ObjectStore.h b/llvm/include/llvm/CAS/ObjectStore.h
index eb61b5f0eb36f..020ca0ac527e8 100644
--- a/llvm/include/llvm/CAS/ObjectStore.h
+++ b/llvm/include/llvm/CAS/ObjectStore.h
@@ -82,7 +82,9 @@ class ActionCache;
/// lifetime tradeoffs:
///
/// - \a getData() accesses data without exposing lifetime at all.
-/// - \a getMemoryBuffer() returns a \a MemoryBuffer whose lifetime
+/// - \a getMemoryBuffer() returns a \a MemoryBuffer that may alias storage
+/// owned by the CAS, so it must not outlive the \a ObjectStore.
+/// - \a getStandaloneMemoryBuffer() returns a \a MemoryBuffer whose lifetime
/// is independent of the CAS (it can live longer).
/// - \a getDataString() return StringRef with lifetime is guaranteed to last as
/// long as \a ObjectStore.
@@ -178,14 +180,24 @@ class LLVM_ABI ObjectStore {
return toStringRef(getData(Node));
}
- /// Get a lifetime-extended MemoryBuffer pointing at \p Data.
+ /// Get a MemoryBuffer pointing at \p Data.
///
- /// Depending on the CAS implementation, this may involve in-memory storage
- /// overhead.
+ /// The buffer may alias storage owned by this ObjectStore, in which case it
+ /// is only valid for as long as the store is.
std::unique_ptr<MemoryBuffer>
getMemoryBuffer(ObjectHandle Node, StringRef Name = "",
bool RequiresNullTerminator = true);
+ /// Get a MemoryBuffer for \p Node that stays valid after this ObjectStore is
+ /// destroyed.
+ ///
+ /// May be more expensive than \a getMemoryBuffer(), which is free to alias
+ /// storage the store already has mapped; prefer that one whenever the buffer
+ /// cannot outlive the store.
+ std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name = "",
+ bool RequiresNullTerminator = true);
+
/// Read all the refs from object in a SmallVector.
virtual void readRefs(ObjectHandle Node,
SmallVectorImpl<ObjectRef> &Refs) const;
@@ -287,6 +299,15 @@ class LLVM_ABI ObjectStore {
ObjectStore(const CASContext &Context) : Context(Context) {}
private:
+ /// Customization point for \a getStandaloneMemoryBuffer(). The default
+ /// implementation copies the data, which always satisfies the lifetime
+ /// requirement; implementations that can hand out storage outliving
+ /// themselves, e.g. a mapping of a file they do not keep open, should
+ /// override this to avoid the copy.
+ virtual std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator);
+
const CASContext &Context;
};
@@ -318,6 +339,11 @@ class ObjectProxy {
getMemoryBuffer(StringRef Name = "",
bool RequiresNullTerminator = true) const;
+ /// Get a MemoryBuffer that stays valid after the CAS is destroyed.
+ LLVM_ABI std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBuffer(StringRef Name = "",
+ bool RequiresNullTerminator = true) const;
+
/// Get the content of the node. Valid as long as the CAS is valid.
StringRef getData() const { return CAS->getDataString(H); }
diff --git a/llvm/include/llvm/CAS/OnDiskGraphDB.h b/llvm/include/llvm/CAS/OnDiskGraphDB.h
index f994a7134a364..a7bf3a123b3f2 100644
--- a/llvm/include/llvm/CAS/OnDiskGraphDB.h
+++ b/llvm/include/llvm/CAS/OnDiskGraphDB.h
@@ -22,6 +22,10 @@
#include "llvm/CAS/OnDiskTrieRawHashMap.h"
#include <atomic>
+namespace llvm {
+class MemoryBuffer;
+} // namespace llvm
+
namespace llvm::cas::ondisk {
/// Standard 8 byte reference inside OnDiskGraphDB.
@@ -354,6 +358,16 @@ class OnDiskGraphDB {
LLVM_ABI FileBackedData
getInternalFileBackedObjectData(ObjectHandle Node) const;
+ /// Get a MemoryBuffer for \p Node's data that stays valid after this
+ /// database is destroyed.
+ ///
+ /// Maps the data where it can, so the pages stay evictable and are not
+ /// charged as dirty memory, and copies where it cannot. Either way the
+ /// result does not reference anything this database owns.
+ LLVM_ABI std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) const;
+
/// \returns Total size of stored objects.
///
/// NOTE: There's a possibility that the returned size is not including a
diff --git a/llvm/lib/CAS/ObjectStore.cpp b/llvm/lib/CAS/ObjectStore.cpp
index e52be9711cac3..cc8013a3fea16 100644
--- a/llvm/lib/CAS/ObjectStore.cpp
+++ b/llvm/lib/CAS/ObjectStore.cpp
@@ -75,6 +75,19 @@ ObjectStore::getMemoryBuffer(ObjectHandle Node, StringRef Name,
RequiresNullTerminator);
}
+std::unique_ptr<MemoryBuffer>
+ObjectStore::getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) {
+ return getStandaloneMemoryBufferImpl(Node, Name, RequiresNullTerminator);
+}
+
+std::unique_ptr<MemoryBuffer>
+ObjectStore::getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) {
+ return MemoryBuffer::getMemBufferCopy(
+ toStringRef(getData(Node, RequiresNullTerminator)), Name);
+}
+
void ObjectStore::readRefs(ObjectHandle Node,
SmallVectorImpl<ObjectRef> &Refs) const {
consumeError(forEachRef(Node, [&Refs](ObjectRef Ref) -> Error {
@@ -294,3 +307,9 @@ ObjectProxy::getMemoryBuffer(StringRef Name,
bool RequiresNullTerminator) const {
return CAS->getMemoryBuffer(H, Name, RequiresNullTerminator);
}
+
+std::unique_ptr<MemoryBuffer>
+ObjectProxy::getStandaloneMemoryBuffer(StringRef Name,
+ bool RequiresNullTerminator) const {
+ return CAS->getStandaloneMemoryBuffer(H, Name, RequiresNullTerminator);
+}
diff --git a/llvm/lib/CAS/OnDiskCAS.cpp b/llvm/lib/CAS/OnDiskCAS.cpp
index 37c2334b2577f..688f75e999dd1 100644
--- a/llvm/lib/CAS/OnDiskCAS.cpp
+++ b/llvm/lib/CAS/OnDiskCAS.cpp
@@ -17,6 +17,7 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/IOSandbox.h"
+#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
using namespace llvm;
@@ -45,6 +46,10 @@ class OnDiskCAS : public BuiltinCAS {
Error exportDataToFile(ObjectHandle Node, StringRef Path) const final;
+ std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) final;
+
void print(raw_ostream &OS) const final;
Error validate(bool CheckHash) const final;
@@ -166,6 +171,13 @@ Expected<ObjectRef> OnDiskCAS::storeFromFile(StringRef Path) {
return convertRef(*StoredID);
}
+std::unique_ptr<MemoryBuffer>
+OnDiskCAS::getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) {
+ return DB->getStandaloneMemoryBuffer(convertHandle(Node), Name,
+ RequiresNullTerminator);
+}
+
Error OnDiskCAS::exportDataToFile(ObjectHandle Node, StringRef Path) const {
auto FBData = DB->getInternalFileBackedObjectData(convertHandle(Node));
if (!FBData.FileInfo.has_value())
diff --git a/llvm/lib/CAS/OnDiskGraphDB.cpp b/llvm/lib/CAS/OnDiskGraphDB.cpp
index c38dd9151bc08..36554ca6a3c7b 100644
--- a/llvm/lib/CAS/OnDiskGraphDB.cpp
+++ b/llvm/lib/CAS/OnDiskGraphDB.cpp
@@ -371,6 +371,12 @@ class StandaloneDataInMemory {
OnDiskGraphDB::FileBackedData
getInternalFileBackedObjectData(StringRef RootPath) const;
+ /// Map this object's file independently of \a Region, so the result stays
+ /// valid after this object is gone. Returns \c nullptr if it cannot be done.
+ std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBuffer(StringRef RootPath, StringRef Name,
+ bool RequiresNullTerminator) const;
+
StandaloneDataInMemory(std::unique_ptr<sys::fs::mapped_file_region> Region,
TrieRecord::StorageKind SK, FileOffset IndexOffset)
: Region(std::move(Region)), SK(SK), IndexOffset(IndexOffset) {
@@ -1291,6 +1297,22 @@ OnDiskGraphDB::getInternalFileBackedObjectData(ObjectHandle Node) const {
}
}
+std::unique_ptr<MemoryBuffer>
+OnDiskGraphDB::getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) const {
+ // Only an object with a file to itself can be mapped; one in the shared data
+ // pool is a subrange of a file holding unrelated objects.
+ auto SDIMOrRecord = getStandaloneDataOrDataRecord(DataPool, Node);
+ if (std::holds_alternative<const StandaloneDataInMemory *>(SDIMOrRecord)) {
+ auto *SDIM = std::get<const StandaloneDataInMemory *>(SDIMOrRecord);
+ if (std::unique_ptr<MemoryBuffer> Mapped = SDIM->getStandaloneMemoryBuffer(
+ RootPath, Name, RequiresNullTerminator))
+ return Mapped;
+ }
+
+ return MemoryBuffer::getMemBufferCopy(toStringRef(getObjectData(Node)), Name);
+}
+
Expected<std::optional<ObjectHandle>>
OnDiskGraphDB::load(ObjectID ExternalRef) {
InternalRef Ref = getInternalRef(ExternalRef);
@@ -1463,6 +1485,62 @@ StandaloneDataInMemory::getInternalFileBackedObjectData(
llvm_unreachable("Unknown StorageKind enum");
}
+namespace {
+/// A MemoryBuffer exposing a subrange of another buffer's bytes, under its own
+/// name.
+class AdoptedMemoryBuffer : public MemoryBuffer {
+public:
+ AdoptedMemoryBuffer(std::unique_ptr<MemoryBuffer> Buffer, StringRef Name,
+ uint64_t Offset, uint64_t Size)
+ : Buffer(std::move(Buffer)), Name(Name.str()) {
+ const char *Start = this->Buffer->getBufferStart() + Offset;
+ init(Start, Start + Size, /*RequiresNullTerminator=*/false);
+ }
+
+ StringRef getBufferIdentifier() const override { return Name; }
+
+ BufferKind getBufferKind() const override { return Buffer->getBufferKind(); }
+
+private:
+ std::unique_ptr<MemoryBuffer> Buffer;
+ std::string Name;
+};
+} // end anonymous namespace
+
+std::unique_ptr<MemoryBuffer> StandaloneDataInMemory::getStandaloneMemoryBuffer(
+ StringRef RootPath, StringRef Name, bool RequiresNullTerminator) const {
+ // A plain leaf's file is exactly the data, with no nul after it to map. The
+ // other kinds have one: a record's own terminator, or the one appended to a
+ // "leaf+0".
+ if (RequiresNullTerminator && SK == TrieRecord::StorageKind::StandaloneLeaf)
+ return nullptr;
+
+ // These files are written once and never resized, and are only deleted along
+ // with the whole directory they live in, which leaves a mapping of them
+ // intact. Map read-only, which is MAP_PRIVATE.
+ SmallString<256> Path;
+ ::getStandalonePath(RootPath, TrieRecord::getStandaloneFilePrefix(SK),
+ IndexOffset, Path);
+ auto BypassSandbox = sys::sandbox::scopedDisable();
+ ErrorOr<std::unique_ptr<MemoryBuffer>> Mapped =
+ MemoryBuffer::getFile(Path, /*IsText=*/false,
+ /*RequiresNullTerminator=*/false,
+ /*IsVolatile=*/false);
+ if (!Mapped)
+ return nullptr;
+
+ // Find the data within the mapping. A leaf's file holds just the data; a
+ // record's also holds its header and refs.
+ OnDiskContent Content = getContent();
+ ArrayRef<char> Data = Content.getData();
+ uint64_t Offset = Content.Record ? Data.data() - Region->data() : 0;
+ if (Offset + Data.size() > (*Mapped)->getBufferSize())
+ return nullptr;
+
+ return std::make_unique<AdoptedMemoryBuffer>(std::move(*Mapped), Name, Offset,
+ Data.size());
+}
+
static Expected<MappedTempFile>
createTempFile(StringRef FinalPath, uint64_t Size, OnDiskCASLogger *Logger) {
auto BypassSandbox = sys::sandbox::scopedDisable();
diff --git a/llvm/unittests/CAS/ObjectStoreTest.cpp b/llvm/unittests/CAS/ObjectStoreTest.cpp
index af248b6de8432..6619cae66d93e 100644
--- a/llvm/unittests/CAS/ObjectStoreTest.cpp
+++ b/llvm/unittests/CAS/ObjectStoreTest.cpp
@@ -8,7 +8,10 @@
#include "llvm/CAS/ObjectStore.h"
#include "OnDiskCommonUtils.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/RandomNumberGenerator.h"
@@ -498,3 +501,148 @@ TEST_F(OnDiskCASTest, OnDiskCASDiskSize) {
CAS.reset();
CheckFileSizes(/*Mapped=*/false);
}
+
+TEST_P(CASTest, StandaloneMemoryBufferOutlivesCAS) {
+ // The buffer has to stay readable after the last reference to the store is
+ // gone. Cover both sides of the size threshold that decides whether an
+ // on-disk CAS embeds an object in its shared data pool or gives it a file of
+ // its own, since only the latter can be mapped.
+ for (uint64_t Size : {64ULL, 100ULL * 1024}) {
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ std::string Data(Size, '\a');
+ Data.front() = 'b';
+ Data.back() = 'e';
+
+ std::optional<ObjectProxy> Proxy;
+ ASSERT_THAT_ERROR(CAS->createProxy({}, Data).moveInto(Proxy), Succeeded());
+ std::unique_ptr<MemoryBuffer> Buffer =
+ Proxy->getStandaloneMemoryBuffer("name");
+ ASSERT_TRUE(Buffer);
+ EXPECT_EQ("name", Buffer->getBufferIdentifier());
+
+ Proxy.reset();
+ CAS.reset();
+
+ // Read every page, not just the ends, so a mapping that lost its backing
+ // store faults here rather than silently passing.
+ ASSERT_EQ(Size, Buffer->getBufferSize());
+ EXPECT_EQ(Data, Buffer->getBuffer());
+ }
+}
+
+TEST_P(CASTest, StandaloneMemoryBufferNullTerminated) {
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ // Cover both sides of the size threshold that decides whether an on-disk CAS
+ // gives an object a file of its own or embeds it in the shared data pool,
+ // and include exact multiples of the page size: a mapping ending on a page
+ // boundary has no zero-filled slack to serve as the terminator.
+ uint64_t PageSize = sys::Process::getPageSizeEstimate();
+ for (uint64_t Size :
+ {64ULL, 60000ULL, 65535ULL, 100ULL * 1024, PageSize, 4 * PageSize}) {
+ std::string Data(Size, 'z');
+ std::optional<ObjectProxy> Proxy;
+ ASSERT_THAT_ERROR(CAS->createProxy({}, Data).moveInto(Proxy), Succeeded());
+ std::unique_ptr<MemoryBuffer> Buffer = Proxy->getStandaloneMemoryBuffer(
+ "name", /*RequiresNullTerminator=*/true);
+ ASSERT_TRUE(Buffer);
+ ASSERT_EQ(Size, Buffer->getBufferSize());
+ EXPECT_EQ('\0', *Buffer->getBufferEnd());
+ EXPECT_EQ(Data, Buffer->getBuffer());
+ }
+}
+
+TEST_P(CASTest, StandaloneMemoryBufferWithoutNullTerminator) {
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ uint64_t PageSize = sys::Process::getPageSizeEstimate();
+ for (uint64_t Size :
+ {64ULL, 60000ULL, 65535ULL, 100ULL * 1024, PageSize, 4 * PageSize}) {
+ std::string Data(Size, 'q');
+ Data.front() = 'b';
+ Data.back() = 'e';
+ std::optional<ObjectProxy> Proxy;
+ ASSERT_THAT_ERROR(CAS->createProxy({}, Data).moveInto(Proxy), Succeeded());
+ std::unique_ptr<MemoryBuffer> Buffer = Proxy->getStandaloneMemoryBuffer(
+ "name", /*RequiresNullTerminator=*/false);
+ ASSERT_TRUE(Buffer);
+ ASSERT_EQ(Size, Buffer->getBufferSize());
+ EXPECT_EQ(Data, Buffer->getBuffer());
+ }
+}
+
+TEST_F(OnDiskCASTest, StandaloneMemoryBufferSurvivesDeletedCAS) {
+ // The point of a standalone buffer is that clients can drop the store,
+ // letting its lock go so the CAS can be pruned, and still read what they
+ // loaded. Deleting the whole directory is the strongest form of that: a
+ // mapping keeps the file alive until it is unmapped.
+ unittest::TempDir Temp("on-disk-cas", /*Unique=*/true);
+
+ std::string SmallData(64, 's');
+ std::string BigData(100ULL * 1024, 'b');
+ BigData.back() = 'e';
+
+ std::unique_ptr<MemoryBuffer> SmallBuffer, BigBuffer;
+ {
+ std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
+ ASSERT_THAT_ERROR(
+ createOnDiskUnifiedCASDatabases(Temp.path()).moveInto(DBs),
+ Succeeded());
+ std::optional<ObjectProxy> Small, Big;
+ ASSERT_THAT_ERROR(DBs.first->createProxy({}, SmallData).moveInto(Small),
+ Succeeded());
+ ASSERT_THAT_ERROR(DBs.first->createProxy({}, BigData).moveInto(Big),
+ Succeeded());
+ SmallBuffer =
+ Small->getStandaloneMemoryBuffer("", /*RequiresNullTerminator=*/false);
+ BigBuffer =
+ Big->getStandaloneMemoryBuffer("", /*RequiresNullTerminator=*/false);
+ ASSERT_TRUE(SmallBuffer);
+ ASSERT_TRUE(BigBuffer);
+ }
+
+ ASSERT_EQ(std::error_code(), sys::fs::remove_directories(Temp.path()));
+ ASSERT_FALSE(sys::fs::exists(Temp.path()));
+
+ EXPECT_EQ(SmallData, SmallBuffer->getBuffer());
+ EXPECT_EQ(BigData, BigBuffer->getBuffer());
+
+ // The big object gets a file of its own, so it should be mapped rather than
+ // copied -- that is the whole point of not holding the store open. (The
+ // small one is embedded in the shared data pool, which is always copied.)
+ EXPECT_EQ(MemoryBuffer::MemoryBuffer_MMap, BigBuffer->getBufferKind());
+ EXPECT_EQ(MemoryBuffer::MemoryBuffer_Malloc, SmallBuffer->getBufferKind());
+}
+
+TEST_F(OnDiskCASTest, StandaloneMemoryBufferRecordWithRefs) {
+ // An object with refs that is too big for the pool gets a file of its own
+ // too, but one holding a record: a header and the refs, then the data and a
+ // nul. It can still be mapped, at an offset, and the nul it already has
+ // means even a caller wanting a terminator does not force a copy.
+ unittest::TempDir Temp("on-disk-cas", /*Unique=*/true);
+
+ std::string Data(100ULL * 1024, 'r');
+ Data.back() = 'e';
+
+ std::unique_ptr<MemoryBuffer> Buffer;
+ {
+ std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
+ ASSERT_THAT_ERROR(
+ createOnDiskUnifiedCASDatabases(Temp.path()).moveInto(DBs),
+ Succeeded());
+ std::optional<ObjectProxy> Child, Parent;
+ ASSERT_THAT_ERROR(DBs.first->createProxy({}, "child").moveInto(Child),
+ Succeeded());
+ ASSERT_THAT_ERROR(
+ DBs.first->createProxy({Child->getRef()}, Data).moveInto(Parent),
+ Succeeded());
+ Buffer = Parent->getStandaloneMemoryBuffer("name");
+ ASSERT_TRUE(Buffer);
+ }
+
+ ASSERT_EQ(std::error_code(), sys::fs::remove_directories(Temp.path()));
+
+ EXPECT_EQ(MemoryBuffer::MemoryBuffer_MMap, Buffer->getBufferKind());
+ EXPECT_EQ("name", Buffer->getBufferIdentifier());
+ ASSERT_EQ(Data.size(), Buffer->getBufferSize());
+ EXPECT_EQ(Data, Buffer->getBuffer());
+ EXPECT_EQ('\0', *Buffer->getBufferEnd());
+}
>From 75a293691728866668383f79c58da5d404e65543 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Mon, 10 Aug 2026 12:15:04 -0700
Subject: [PATCH 2/4] fix linux and windows test failures
Created using spr 1.3.7
---
llvm/unittests/CAS/ObjectStoreTest.cpp | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/llvm/unittests/CAS/ObjectStoreTest.cpp b/llvm/unittests/CAS/ObjectStoreTest.cpp
index 6619cae66d93e..aff7f783ee9ab 100644
--- a/llvm/unittests/CAS/ObjectStoreTest.cpp
+++ b/llvm/unittests/CAS/ObjectStoreTest.cpp
@@ -507,7 +507,7 @@ TEST_P(CASTest, StandaloneMemoryBufferOutlivesCAS) {
// gone. Cover both sides of the size threshold that decides whether an
// on-disk CAS embeds an object in its shared data pool or gives it a file of
// its own, since only the latter can be mapped.
- for (uint64_t Size : {64ULL, 100ULL * 1024}) {
+ for (uint64_t Size : {uint64_t(64), uint64_t(100 * 1024)}) {
std::shared_ptr<ObjectStore> CAS = createObjectStore();
std::string Data(Size, '\a');
Data.front() = 'b';
@@ -537,8 +537,8 @@ TEST_P(CASTest, StandaloneMemoryBufferNullTerminated) {
// and include exact multiples of the page size: a mapping ending on a page
// boundary has no zero-filled slack to serve as the terminator.
uint64_t PageSize = sys::Process::getPageSizeEstimate();
- for (uint64_t Size :
- {64ULL, 60000ULL, 65535ULL, 100ULL * 1024, PageSize, 4 * PageSize}) {
+ for (uint64_t Size : {uint64_t(64), uint64_t(60000), uint64_t(65535),
+ uint64_t(100 * 1024), PageSize, 4 * PageSize}) {
std::string Data(Size, 'z');
std::optional<ObjectProxy> Proxy;
ASSERT_THAT_ERROR(CAS->createProxy({}, Data).moveInto(Proxy), Succeeded());
@@ -554,8 +554,8 @@ TEST_P(CASTest, StandaloneMemoryBufferNullTerminated) {
TEST_P(CASTest, StandaloneMemoryBufferWithoutNullTerminator) {
std::shared_ptr<ObjectStore> CAS = createObjectStore();
uint64_t PageSize = sys::Process::getPageSizeEstimate();
- for (uint64_t Size :
- {64ULL, 60000ULL, 65535ULL, 100ULL * 1024, PageSize, 4 * PageSize}) {
+ for (uint64_t Size : {uint64_t(64), uint64_t(60000), uint64_t(65535),
+ uint64_t(100 * 1024), PageSize, 4 * PageSize}) {
std::string Data(Size, 'q');
Data.front() = 'b';
Data.back() = 'e';
@@ -599,8 +599,12 @@ TEST_F(OnDiskCASTest, StandaloneMemoryBufferSurvivesDeletedCAS) {
ASSERT_TRUE(BigBuffer);
}
+ // Windows refuses to delete a file while it is still mapped, so only check
+ // the deleted case where unlinking a mapped file is allowed.
+#ifndef _WIN32
ASSERT_EQ(std::error_code(), sys::fs::remove_directories(Temp.path()));
ASSERT_FALSE(sys::fs::exists(Temp.path()));
+#endif
EXPECT_EQ(SmallData, SmallBuffer->getBuffer());
EXPECT_EQ(BigData, BigBuffer->getBuffer());
@@ -638,8 +642,6 @@ TEST_F(OnDiskCASTest, StandaloneMemoryBufferRecordWithRefs) {
ASSERT_TRUE(Buffer);
}
- ASSERT_EQ(std::error_code(), sys::fs::remove_directories(Temp.path()));
-
EXPECT_EQ(MemoryBuffer::MemoryBuffer_MMap, Buffer->getBufferKind());
EXPECT_EQ("name", Buffer->getBufferIdentifier());
ASSERT_EQ(Data.size(), Buffer->getBufferSize());
>From 68897daa0160733ba5abee9356024b2ae3c069d6 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Mon, 10 Aug 2026 13:04:02 -0700
Subject: [PATCH 3/4] address review feedback.
Created using spr 1.3.7
---
llvm/docs/ContentAddressableStorage.md | 4 +--
llvm/include/llvm/CAS/ObjectStore.h | 7 ++++--
llvm/include/llvm/CAS/OnDiskGraphDB.h | 7 +++---
llvm/lib/CAS/OnDiskGraphDB.cpp | 35 +++++++++++++++-----------
4 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/llvm/docs/ContentAddressableStorage.md b/llvm/docs/ContentAddressableStorage.md
index ea8c6a2d2e39b..702c48bac3169 100644
--- a/llvm/docs/ContentAddressableStorage.md
+++ b/llvm/docs/ContentAddressableStorage.md
@@ -64,8 +64,8 @@ the lifetime of the instance of `ObjectStore` unless explicitly copied.
To get a lifetime-extended buffer, call `getStandaloneMemoryBuffer()`, which
returns a `MemoryBuffer` that remains valid after the `ObjectStore` is
destroyed. A CAS can provide a customized implementation that is cheaper than
-a copy; the on-disk CAS maps the object's file where it has one, so the pages
-stay evictable instead of being charged as dirty memory.
+a copy; the on-disk CAS re-reads the object's file where it has one, which
+lets the pages be shared and reclaimed rather than charged to the process.
### CASID
diff --git a/llvm/include/llvm/CAS/ObjectStore.h b/llvm/include/llvm/CAS/ObjectStore.h
index 020ca0ac527e8..1e2a4ed685ff6 100644
--- a/llvm/include/llvm/CAS/ObjectStore.h
+++ b/llvm/include/llvm/CAS/ObjectStore.h
@@ -193,7 +193,8 @@ class LLVM_ABI ObjectStore {
///
/// May be more expensive than \a getMemoryBuffer(), which is free to alias
/// storage the store already has mapped; prefer that one whenever the buffer
- /// cannot outlive the store.
+ /// cannot outlive the store. Never returns \c nullptr: copying the data
+ /// always satisfies the lifetime requirement.
std::unique_ptr<MemoryBuffer>
getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name = "",
bool RequiresNullTerminator = true);
@@ -303,7 +304,9 @@ class LLVM_ABI ObjectStore {
/// implementation copies the data, which always satisfies the lifetime
/// requirement; implementations that can hand out storage outliving
/// themselves, e.g. a mapping of a file they do not keep open, should
- /// override this to avoid the copy.
+ /// override this to avoid the copy. Must not return \c nullptr: fall back
+ /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
+ /// path does not apply.
virtual std::unique_ptr<MemoryBuffer>
getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
bool RequiresNullTerminator);
diff --git a/llvm/include/llvm/CAS/OnDiskGraphDB.h b/llvm/include/llvm/CAS/OnDiskGraphDB.h
index a7bf3a123b3f2..addc39a85598f 100644
--- a/llvm/include/llvm/CAS/OnDiskGraphDB.h
+++ b/llvm/include/llvm/CAS/OnDiskGraphDB.h
@@ -361,9 +361,10 @@ class OnDiskGraphDB {
/// Get a MemoryBuffer for \p Node's data that stays valid after this
/// database is destroyed.
///
- /// Maps the data where it can, so the pages stay evictable and are not
- /// charged as dirty memory, and copies where it cannot. Either way the
- /// result does not reference anything this database owns.
+ /// Objects stored in a file of their own are re-read from it rather than
+ /// copied out of this database's mapping, which lets the pages be shared and
+ /// reclaimed rather than charged to this process. The rest are copied. Never
+ /// returns \c nullptr.
LLVM_ABI std::unique_ptr<MemoryBuffer>
getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name,
bool RequiresNullTerminator) const;
diff --git a/llvm/lib/CAS/OnDiskGraphDB.cpp b/llvm/lib/CAS/OnDiskGraphDB.cpp
index 36554ca6a3c7b..a817b35f6f588 100644
--- a/llvm/lib/CAS/OnDiskGraphDB.cpp
+++ b/llvm/lib/CAS/OnDiskGraphDB.cpp
@@ -371,8 +371,11 @@ class StandaloneDataInMemory {
OnDiskGraphDB::FileBackedData
getInternalFileBackedObjectData(StringRef RootPath) const;
- /// Map this object's file independently of \a Region, so the result stays
- /// valid after this object is gone. Returns \c nullptr if it cannot be done.
+ /// Read this object's data from its file again, so the result does not
+ /// reference \a Region and stays valid after this object is gone.
+ ///
+ /// \returns \c nullptr when it does not apply, and the caller is
+ /// expected to copy instead.
std::unique_ptr<MemoryBuffer>
getStandaloneMemoryBuffer(StringRef RootPath, StringRef Name,
bool RequiresNullTerminator) const;
@@ -1300,14 +1303,15 @@ OnDiskGraphDB::getInternalFileBackedObjectData(ObjectHandle Node) const {
std::unique_ptr<MemoryBuffer>
OnDiskGraphDB::getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name,
bool RequiresNullTerminator) const {
- // Only an object with a file to itself can be mapped; one in the shared data
- // pool is a subrange of a file holding unrelated objects.
+ // Only an object with a file to itself can be read back on its own; one in
+ // the shared data pool is a subrange of a file holding unrelated objects.
auto SDIMOrRecord = getStandaloneDataOrDataRecord(DataPool, Node);
- if (std::holds_alternative<const StandaloneDataInMemory *>(SDIMOrRecord)) {
- auto *SDIM = std::get<const StandaloneDataInMemory *>(SDIMOrRecord);
- if (std::unique_ptr<MemoryBuffer> Mapped = SDIM->getStandaloneMemoryBuffer(
- RootPath, Name, RequiresNullTerminator))
- return Mapped;
+ if (auto **SDIM =
+ std::get_if<const StandaloneDataInMemory *>(&SDIMOrRecord)) {
+ if (std::unique_ptr<MemoryBuffer> Standalone =
+ (*SDIM)->getStandaloneMemoryBuffer(RootPath, Name,
+ RequiresNullTerminator))
+ return Standalone;
}
return MemoryBuffer::getMemBufferCopy(toStringRef(getObjectData(Node)), Name);
@@ -1488,7 +1492,7 @@ StandaloneDataInMemory::getInternalFileBackedObjectData(
namespace {
/// A MemoryBuffer exposing a subrange of another buffer's bytes, under its own
/// name.
-class AdoptedMemoryBuffer : public MemoryBuffer {
+class AdoptedMemoryBuffer final : public MemoryBuffer {
public:
AdoptedMemoryBuffer(std::unique_ptr<MemoryBuffer> Buffer, StringRef Name,
uint64_t Offset, uint64_t Size)
@@ -1497,9 +1501,9 @@ class AdoptedMemoryBuffer : public MemoryBuffer {
init(Start, Start + Size, /*RequiresNullTerminator=*/false);
}
- StringRef getBufferIdentifier() const override { return Name; }
+ StringRef getBufferIdentifier() const final { return Name; }
- BufferKind getBufferKind() const override { return Buffer->getBufferKind(); }
+ BufferKind getBufferKind() const final { return Buffer->getBufferKind(); }
private:
std::unique_ptr<MemoryBuffer> Buffer;
@@ -1515,9 +1519,10 @@ std::unique_ptr<MemoryBuffer> StandaloneDataInMemory::getStandaloneMemoryBuffer(
if (RequiresNullTerminator && SK == TrieRecord::StorageKind::StandaloneLeaf)
return nullptr;
- // These files are written once and never resized, and are only deleted along
- // with the whole directory they live in, which leaves a mapping of them
- // intact. Map read-only, which is MAP_PRIVATE.
+ // Read the file again instead of sharing \a Region, whose lifetime is tied
+ // to this object. These files are written once and never modified, so the
+ // second read sees the same bytes. Whether that ends up mapping the file or
+ // copying it is up to MemoryBuffer; either way the result stands alone.
SmallString<256> Path;
::getStandalonePath(RootPath, TrieRecord::getStandaloneFilePrefix(SK),
IndexOffset, Path);
>From c4f7c7770ad278c145bbedade923bd15b1391af0 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Tue, 11 Aug 2026 12:29:52 -0700
Subject: [PATCH 4/4] add plugin CAS impl
Created using spr 1.3.7
---
llvm/include/llvm-c/CAS/PluginAPI_functions.h | 23 +++++++++
llvm/include/llvm-c/CAS/PluginAPI_types.h | 2 +-
llvm/include/llvm/CAS/ObjectStore.h | 22 ++++----
llvm/lib/CAS/PluginAPI.h | 5 ++
llvm/lib/CAS/PluginAPI_functions.def | 2 +
llvm/lib/CAS/PluginCAS.cpp | 51 +++++++++++++++++++
.../libCASPluginTest/libCASPluginTest.cpp | 36 +++++++++++++
.../libCASPluginTest/libCASPluginTest.exports | 2 +
8 files changed, 131 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm-c/CAS/PluginAPI_functions.h b/llvm/include/llvm-c/CAS/PluginAPI_functions.h
index 1102d094213b6..94ef0759873b3 100644
--- a/llvm/include/llvm-c/CAS/PluginAPI_functions.h
+++ b/llvm/include/llvm-c/CAS/PluginAPI_functions.h
@@ -305,6 +305,29 @@ LLCAS_PUBLIC bool llcas_cas_store_from_filepath(llcas_cas_t,
LLCAS_PUBLIC llcas_data_t llcas_loaded_object_get_data(llcas_cas_t,
llcas_loaded_object_t);
+/**
+ * \returns a data buffer for the provided \c llcas_loaded_object_t that stays
+ * valid after the \c llcas_cas_t is disposed of. The buffer pointer must be
+ * 8-byte aligned and \c NULL terminated. It must be released via
+ * \c llcas_standalone_data_dispose, which may outlive the \c llcas_cas_t.
+ *
+ * This is an optimization over copying the buffer returned by
+ * \c llcas_loaded_object_get_data: an implementation that can hand out storage
+ * outliving itself, e.g. a mapping of a file it does not keep open, avoids the
+ * copy. Implementing it is optional, and requires
+ * \c llcas_standalone_data_dispose to be implemented as well.
+ */
+LLCAS_PUBLIC llcas_data_t
+ llcas_loaded_object_get_standalone_data(llcas_cas_t, llcas_loaded_object_t);
+
+/**
+ * Releases a buffer returned by \c llcas_loaded_object_get_standalone_data.
+ *
+ * This may be called after the \c llcas_cas_t that produced the buffer has
+ * been disposed of, so it must not depend on it.
+ */
+LLCAS_PUBLIC void llcas_standalone_data_dispose(llcas_data_t);
+
/**
* \returns the references of the provided \c llcas_loaded_object_t.
*/
diff --git a/llvm/include/llvm-c/CAS/PluginAPI_types.h b/llvm/include/llvm-c/CAS/PluginAPI_types.h
index 6f969837446a4..2d45461aea28a 100644
--- a/llvm/include/llvm-c/CAS/PluginAPI_types.h
+++ b/llvm/include/llvm-c/CAS/PluginAPI_types.h
@@ -20,7 +20,7 @@
#include <stdint.h>
#define LLCAS_VERSION_MAJOR 0
-#define LLCAS_VERSION_MINOR 1
+#define LLCAS_VERSION_MINOR 2
typedef struct llcas_cas_options_s *llcas_cas_options_t;
typedef struct llcas_cas_s *llcas_cas_t;
diff --git a/llvm/include/llvm/CAS/ObjectStore.h b/llvm/include/llvm/CAS/ObjectStore.h
index 1e2a4ed685ff6..972d8eb02b724 100644
--- a/llvm/include/llvm/CAS/ObjectStore.h
+++ b/llvm/include/llvm/CAS/ObjectStore.h
@@ -172,6 +172,17 @@ class LLVM_ABI ObjectStore {
storeFromOpenFileImpl(sys::fs::file_t FD,
std::optional<sys::fs::file_status> Status);
+ /// Customization point for \a getStandaloneMemoryBuffer(). The default
+ /// implementation copies the data, which always satisfies the lifetime
+ /// requirement; implementations that can hand out storage outliving
+ /// themselves, e.g. a mapping of a file they do not keep open, should
+ /// override this to avoid the copy. Must not return \c nullptr: fall back
+ /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
+ /// path does not apply.
+ virtual std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator);
+
/// Get a lifetime-extended StringRef pointing at \p Data.
///
/// Depending on the CAS implementation, this may involve in-memory storage
@@ -300,17 +311,6 @@ class LLVM_ABI ObjectStore {
ObjectStore(const CASContext &Context) : Context(Context) {}
private:
- /// Customization point for \a getStandaloneMemoryBuffer(). The default
- /// implementation copies the data, which always satisfies the lifetime
- /// requirement; implementations that can hand out storage outliving
- /// themselves, e.g. a mapping of a file they do not keep open, should
- /// override this to avoid the copy. Must not return \c nullptr: fall back
- /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
- /// path does not apply.
- virtual std::unique_ptr<MemoryBuffer>
- getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
- bool RequiresNullTerminator);
-
const CASContext &Context;
};
diff --git a/llvm/lib/CAS/PluginAPI.h b/llvm/lib/CAS/PluginAPI.h
index 3a694ddca9ec4..769020403464b 100644
--- a/llvm/lib/CAS/PluginAPI.h
+++ b/llvm/lib/CAS/PluginAPI.h
@@ -88,6 +88,11 @@ struct llcas_functions_t {
llcas_object_refs_t (*loaded_object_get_refs)(llcas_cas_t,
llcas_loaded_object_t);
+ llcas_data_t (*loaded_object_get_standalone_data)(llcas_cas_t,
+ llcas_loaded_object_t);
+
+ void (*standalone_data_dispose)(llcas_data_t);
+
size_t (*object_refs_get_count)(llcas_cas_t, llcas_object_refs_t);
llcas_objectid_t (*object_refs_get_id)(llcas_cas_t, llcas_object_refs_t,
diff --git a/llvm/lib/CAS/PluginAPI_functions.def b/llvm/lib/CAS/PluginAPI_functions.def
index 996de2f4c7513..8e779b68d8288 100644
--- a/llvm/lib/CAS/PluginAPI_functions.def
+++ b/llvm/lib/CAS/PluginAPI_functions.def
@@ -46,7 +46,9 @@ CASPLUGINAPI_FUNCTION(get_plugin_version, true)
CASPLUGINAPI_FUNCTION(loaded_object_export_data_to_filepath, false)
CASPLUGINAPI_FUNCTION(loaded_object_get_data, true)
CASPLUGINAPI_FUNCTION(loaded_object_get_refs, true)
+CASPLUGINAPI_FUNCTION(loaded_object_get_standalone_data, false)
CASPLUGINAPI_FUNCTION(object_refs_get_count, true)
CASPLUGINAPI_FUNCTION(object_refs_get_id, true)
CASPLUGINAPI_FUNCTION(objectid_get_digest, true)
+CASPLUGINAPI_FUNCTION(standalone_data_dispose, false)
CASPLUGINAPI_FUNCTION(string_dispose, true)
diff --git a/llvm/lib/CAS/PluginCAS.cpp b/llvm/lib/CAS/PluginCAS.cpp
index 3b38a9d4fdb21..945a5ac43f56e 100644
--- a/llvm/lib/CAS/PluginCAS.cpp
+++ b/llvm/lib/CAS/PluginCAS.cpp
@@ -22,6 +22,7 @@
#include "llvm/CAS/ObjectStore.h"
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
using namespace llvm;
using namespace llvm::cas;
@@ -149,6 +150,9 @@ class PluginObjectStore : public ObjectStore {
size_t getNumRefs(ObjectHandle Node) const final;
ArrayRef<char> getData(ObjectHandle Node,
bool RequiresNullTerminator = false) const final;
+ std::unique_ptr<MemoryBuffer>
+ getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
+ bool RequiresNullTerminator) final;
Error validateObject(const CASID &ID) final {
// Not supported yet. Always return success.
return Error::success();
@@ -372,6 +376,53 @@ ArrayRef<char> PluginObjectStore::getData(ObjectHandle Node,
return ArrayRef((const char *)c_data.data, c_data.size);
}
+namespace {
+/// A MemoryBuffer over a plugin's standalone buffer, which it releases when
+/// destroyed. It holds the dispose function directly rather than the
+/// \c llcas_cas_t, since the point of the buffer is to outlive that.
+class PluginStandaloneMemoryBuffer final : public MemoryBuffer {
+public:
+ using DisposeFn = void (*)(llcas_data_t);
+
+ PluginStandaloneMemoryBuffer(llcas_data_t Data, StringRef Name,
+ DisposeFn Dispose)
+ : Data(Data), Name(Name.str()), Dispose(Dispose) {
+ const char *Start = static_cast<const char *>(Data.data);
+ init(Start, Start + Data.size, /*RequiresNullTerminator=*/true);
+ }
+
+ ~PluginStandaloneMemoryBuffer() override { Dispose(Data); }
+
+ StringRef getBufferIdentifier() const final { return Name; }
+
+ BufferKind getBufferKind() const final { return MemoryBuffer_Malloc; }
+
+private:
+ llcas_data_t Data;
+ std::string Name;
+ DisposeFn Dispose;
+};
+} // namespace
+
+std::unique_ptr<MemoryBuffer> PluginObjectStore::getStandaloneMemoryBufferImpl(
+ ObjectHandle Node, StringRef Name, bool RequiresNullTerminator) {
+ // Both halves are needed: without the disposer there is no way to release
+ // what the getter hands out.
+ if (!Ctx->Functions.loaded_object_get_standalone_data ||
+ !Ctx->Functions.standalone_data_dispose)
+ return ObjectStore::getStandaloneMemoryBufferImpl(Node, Name,
+ RequiresNullTerminator);
+
+ llcas_data_t c_data = Ctx->Functions.loaded_object_get_standalone_data(
+ Ctx->c_cas, llcas_loaded_object_t{Node.getInternalRef(*this)});
+ if (!c_data.data)
+ return ObjectStore::getStandaloneMemoryBufferImpl(Node, Name,
+ RequiresNullTerminator);
+
+ return std::make_unique<PluginStandaloneMemoryBuffer>(
+ c_data, Name, Ctx->Functions.standalone_data_dispose);
+}
+
Error PluginObjectStore::setSizeLimit(std::optional<uint64_t> SizeLimit) {
if (Ctx->Functions.cas_set_ondisk_size_limit) {
char *c_err = nullptr;
diff --git a/llvm/tools/libCASPluginTest/libCASPluginTest.cpp b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp
index 04402bfdaca0f..e024a0b25eaca 100644
--- a/llvm/tools/libCASPluginTest/libCASPluginTest.cpp
+++ b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp
@@ -641,6 +641,42 @@ llcas_data_t llcas_loaded_object_get_data(llcas_cas_t c_cas,
return llcas_data_t{Data.data(), Data.size()};
}
+/// The \c MemoryBuffer objects handed out by
+/// \c llcas_loaded_object_get_standalone_data, keyed by the bytes the C API
+/// reports, so \c llcas_standalone_data_dispose can find the owner again. The
+/// C API passes back only the buffer, and these outlive the \c llcas_cas_t, so
+/// they cannot be tracked on it.
+/// Intentionally leaked, since a buffer may be disposed of during static
+/// destruction, after a non-leaked map would already be gone.
+static std::mutex StandaloneBuffersLock;
+static auto *StandaloneBuffers =
+ new DenseMap<const void *, std::unique_ptr<MemoryBuffer>>();
+
+llcas_data_t
+llcas_loaded_object_get_standalone_data(llcas_cas_t c_cas,
+ llcas_loaded_object_t c_obj) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ ondisk::ObjectHandle Obj = ondisk::ObjectHandle(c_obj.opaque);
+ // The underlying database already knows how to produce a buffer that does
+ // not reference it, so use that rather than copying the data again. The
+ // plugin API requires a nul terminator, which costs a copy for the objects
+ // whose file has no byte to spare for one.
+ std::unique_ptr<MemoryBuffer> Buffer = CAS.getStandaloneMemoryBuffer(
+ Obj, /*Name=*/"", /*RequiresNullTerminator=*/true);
+ const char *Data = Buffer->getBufferStart();
+ size_t Size = Buffer->getBufferSize();
+ {
+ std::lock_guard<std::mutex> Lock(StandaloneBuffersLock);
+ (*StandaloneBuffers)[Data] = std::move(Buffer);
+ }
+ return llcas_data_t{Data, Size};
+}
+
+void llcas_standalone_data_dispose(llcas_data_t c_data) {
+ std::lock_guard<std::mutex> Lock(StandaloneBuffersLock);
+ StandaloneBuffers->erase(c_data.data);
+}
+
llcas_object_refs_t llcas_loaded_object_get_refs(llcas_cas_t c_cas,
llcas_loaded_object_t c_obj) {
auto &CAS = unwrap(c_cas)->DB->getGraphDB();
diff --git a/llvm/tools/libCASPluginTest/libCASPluginTest.exports b/llvm/tools/libCASPluginTest/libCASPluginTest.exports
index ad8bed6c6a689..1c292e4da1ccf 100644
--- a/llvm/tools/libCASPluginTest/libCASPluginTest.exports
+++ b/llvm/tools/libCASPluginTest/libCASPluginTest.exports
@@ -25,7 +25,9 @@ llcas_digest_print
llcas_get_plugin_version
llcas_loaded_object_get_data
llcas_loaded_object_get_refs
+llcas_loaded_object_get_standalone_data
llcas_object_refs_get_count
llcas_object_refs_get_id
llcas_objectid_get_digest
+llcas_standalone_data_dispose
llcas_string_dispose
More information about the llvm-commits
mailing list