[llvm] 1cf7984 - [CAS] Add a plugin C API for providing ObjectStore/ActionCache implementations (#213331)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 10:58:21 PDT 2026
Author: Steven Wu
Date: 2026-08-07T10:58:17-07:00
New Revision: 1cf7984259827c3fc58b340a51feaeec7e2894fd
URL: https://github.com/llvm/llvm-project/commit/1cf7984259827c3fc58b340a51feaeec7e2894fd
DIFF: https://github.com/llvm/llvm-project/commit/1cf7984259827c3fc58b340a51feaeec7e2894fd.diff
LOG: [CAS] Add a plugin C API for providing ObjectStore/ActionCache implementations (#213331)
Add a C API that lets an external library provide a CAS implementation,
along
with an LLVM-side adapter that exposes such a plugin as an
ObjectStore/ActionCache pair.
The pieces are:
- llvm-c/CAS/PluginAPI_types.h and llvm-c/CAS/PluginAPI_functions.h: the
C API
that a plugin implements. Versioned via LLCAS_VERSION_{MAJOR,MINOR};
both the
client and the plugin exchange their versions so implementations can
stay
compatible across changes.
- lib/CAS/PluginAPI.h and lib/CAS/PluginAPI_functions.def: the table of
function pointers resolved from the loaded library. The .def file
records
whether each symbol is required, so optional functionality (storage size
reporting, pruning, validation, file-based store/export) can be omitted
by a
plugin and is then feature-detected at runtime.
- lib/CAS/PluginCAS.cpp: PluginObjectStore and PluginActionCache,
created via
the new cas::createPluginCASDatabases().
- tools/libCASPluginTest: a mock implementation of the C API, backed by
UnifiedOnDiskCache. It can be pointed at a second on-disk path to
simulate
"uploading"/"downloading" objects to/from a distributed CAS, which is
what
lets the distributed code paths be exercised in-tree. Because it is only
ever
used for testing, it caps its on-disk mappings at a small size; it links
its
own copy of LLVMCAS, so the limit a test binary sets for itself does not
reach it.
- unittests/CAS/PluginCASTest.cpp: covers loading the plugin and the
materialization behavior when a key is found remotely but its node graph
is
only faulted in lazily.
- A PluginCAS instantiation of the shared CASTest suite, so the plugin
is run
against the same ObjectStore and ActionCache tests as the in-memory and
on-disk implementations. CASTestingEnv now holds shared_ptr, since
createPluginCASDatabases() hands out shared ownership of the underlying
plugin instance.
The plugin is only built when LLVM_ENABLE_ONDISK_CAS is enabled, as the
mock
implementation is built on UnifiedOnDiskCache.
Added:
llvm/include/llvm-c/CAS/PluginAPI_functions.h
llvm/include/llvm-c/CAS/PluginAPI_types.h
llvm/lib/CAS/PluginAPI.h
llvm/lib/CAS/PluginAPI_functions.def
llvm/lib/CAS/PluginCAS.cpp
llvm/tools/libCASPluginTest/CMakeLists.txt
llvm/tools/libCASPluginTest/libCASPluginTest.cpp
llvm/tools/libCASPluginTest/libCASPluginTest.exports
llvm/unittests/CAS/PluginCASTest.cpp
Modified:
llvm/include/llvm/CAS/ObjectStore.h
llvm/lib/CAS/CMakeLists.txt
llvm/unittests/CAS/ActionCacheTest.cpp
llvm/unittests/CAS/CASTestConfig.cpp
llvm/unittests/CAS/CASTestConfig.h
llvm/unittests/CAS/CMakeLists.txt
llvm/unittests/CAS/ObjectStoreTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm-c/CAS/PluginAPI_functions.h b/llvm/include/llvm-c/CAS/PluginAPI_functions.h
new file mode 100644
index 0000000000000..1102d094213b6
--- /dev/null
+++ b/llvm/include/llvm-c/CAS/PluginAPI_functions.h
@@ -0,0 +1,416 @@
+/*===----------------------------------------------------------------------===*\
+|* *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *|
+|* Exceptions. *|
+|* See https://llvm.org/LICENSE.txt for license information. *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *|
+|* *|
+|*===----------------------------------------------------------------------===*|
+|* *|
+|* The functions for the LLVM CAS plugin API. Intended for assisting *|
+|* implementations of the API. *|
+|* The API is experimental and subject to change. *|
+|* *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_C_CAS_PLUGINAPI_FUNCTIONS_H
+#define LLVM_C_CAS_PLUGINAPI_FUNCTIONS_H
+
+#include "llvm-c/CAS/PluginAPI_types.h"
+#include "llvm-c/ExternC.h"
+
+#ifndef LLCAS_PUBLIC
+#ifdef _WIN32
+#define LLCAS_PUBLIC __declspec(dllexport)
+#else
+#define LLCAS_PUBLIC
+#endif
+#endif
+
+LLVM_C_EXTERN_C_BEGIN
+
+/**
+ * Returns the \c LLCAS_VERSION_MAJOR and \c LLCAS_VERSION_MINOR values that the
+ * plugin was compiled with.
+ * Intended for assisting compatibility with
diff erent versions.
+ */
+LLCAS_PUBLIC void llcas_get_plugin_version(unsigned *major, unsigned *minor);
+
+/**
+ * Releases memory of C string pointers provided by other functions.
+ */
+LLCAS_PUBLIC void llcas_string_dispose(char *);
+
+/**
+ * Cancels the asynchronous query associated with the \c llcas_cancellable_t.
+ */
+LLCAS_PUBLIC void llcas_cancellable_cancel(llcas_cancellable_t);
+
+/**
+ * Releases memory associated with given \c llcas_cancellable_t.
+ */
+LLCAS_PUBLIC void llcas_cancellable_dispose(llcas_cancellable_t);
+
+/**
+ * Options object to configure creation of \c llcas_cas_t. After passing to
+ * \c llcas_cas_create, its memory can be released via
+ * \c llcas_cas_options_dispose.
+ */
+LLCAS_PUBLIC llcas_cas_options_t llcas_cas_options_create(void);
+
+LLCAS_PUBLIC void llcas_cas_options_dispose(llcas_cas_options_t);
+
+/**
+ * Receives the \c LLCAS_VERSION_MAJOR and \c LLCAS_VERSION_MINOR values that
+ * the client was compiled with.
+ * Intended for assisting compatibility with
diff erent versions.
+ */
+LLCAS_PUBLIC void llcas_cas_options_set_client_version(llcas_cas_options_t,
+ unsigned major,
+ unsigned minor);
+
+/**
+ * Receives a local file-system path that the plugin should use for any on-disk
+ * resources/caches.
+ */
+LLCAS_PUBLIC void llcas_cas_options_set_ondisk_path(llcas_cas_options_t,
+ const char *path);
+
+/**
+ * Receives a name/value strings pair, for the plugin to set as a custom option
+ * it supports. These are usually passed through as invocation options and are
+ * opaque to the client.
+ *
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_options_set_option(llcas_cas_options_t,
+ const char *name,
+ const char *value, char **error);
+
+/**
+ * Creates a new \c llcas_cas_t object. The objects returned from the other
+ * functions are only valid to use while the \c llcas_cas_t object that they
+ * came from is still valid.
+ *
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns \c NULL if there was an error.
+ */
+LLCAS_PUBLIC llcas_cas_t llcas_cas_create(llcas_cas_options_t, char **error);
+
+/**
+ * Releases memory of \c llcas_cas_t. After calling this it is invalid to keep
+ * using objects that originated from this \c llcas_cas_t instance.
+ */
+LLCAS_PUBLIC void llcas_cas_dispose(llcas_cas_t);
+
+/**
+ * Get the local storage size of the CAS/cache data in bytes.
+ *
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns the local storage size of the CAS/cache data, or -1 if the
+ * implementation does not support reporting such size, or -2 if an error
+ * occurred.
+ */
+LLCAS_PUBLIC int64_t llcas_cas_get_ondisk_size(llcas_cas_t, char **error);
+
+/**
+ * Set the size for limiting disk storage growth.
+ *
+ * \param size_limit the maximum size limit in bytes. 0 means no limit. Negative
+ * values are invalid.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool
+llcas_cas_set_ondisk_size_limit(llcas_cas_t, int64_t size_limit, char **error);
+
+/**
+ * Prune local storage to reduce its size according to the desired size limit.
+ * Pruning can happen concurrently with other operations.
+ *
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_prune_ondisk_data(llcas_cas_t, char **error);
+
+/**
+ * Validate the CAS contents.
+ *
+ * \param check_hash if true, the hash of each object is recomputed and compared
+ * against the one it is stored under.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_validate(llcas_cas_t, bool check_hash,
+ char **error);
+
+/**
+ * \returns the hash schema name that the plugin is using. The string memory it
+ * points to needs to be released via \c llcas_string_dispose.
+ */
+LLCAS_PUBLIC char *llcas_cas_get_hash_schema_name(llcas_cas_t);
+
+/**
+ * Parses the printed digest and returns the digest hash bytes.
+ *
+ * \param printed_digest a C string that was previously provided by
+ * \c llcas_digest_print.
+ * \param bytes pointer to a buffer for writing the digest bytes. Can be \c NULL
+ * if \p bytes_size is 0.
+ * \param bytes_size the size of the buffer.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns 0 if there was an error. If \p bytes_size is smaller than the
+ * required size to fit the digest bytes, returns the required buffer size
+ * without writing to \c bytes. Otherwise writes the digest bytes to \p bytes
+ * and returns the number of written bytes.
+ */
+LLCAS_PUBLIC unsigned llcas_digest_parse(llcas_cas_t,
+ const char *printed_digest,
+ uint8_t *bytes, size_t bytes_size,
+ char **error);
+
+/**
+ * Returns a string for the given digest bytes that can be passed to
+ * \c llcas_digest_parse.
+ *
+ * \param printed_id pointer to receive the printed digest string. The memory it
+ * points to needs to be released via \c llcas_string_dispose.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_digest_print(llcas_cas_t, llcas_digest_t,
+ char **printed_id, char **error);
+
+/**
+ * Provides the \c llcas_objectid_t value for the given \c llcas_digest_t.
+ *
+ * \param digest the digest bytes that the returned \c llcas_objectid_t
+ * represents.
+ * \param p_id pointer to store the returned \c llcas_objectid_t object.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_get_objectid(llcas_cas_t, llcas_digest_t digest,
+ llcas_objectid_t *p_id, char **error);
+
+/**
+ * \returns the \c llcas_digest_t value for the given \c llcas_objectid_t.
+ * The memory that the buffer points to is valid for the lifetime of the
+ * \c llcas_cas_t object.
+ */
+LLCAS_PUBLIC llcas_digest_t llcas_objectid_get_digest(llcas_cas_t,
+ llcas_objectid_t);
+
+/**
+ * Checks whether a \c llcas_objectid_t points to an existing object.
+ *
+ * \param globally For CAS implementations that distinguish between local CAS
+ * and remote/distributed CAS, \p globally set to false indicates that the
+ * lookup will be restricted to the local CAS, returning "not found" even if the
+ * object might exist in the remote CAS.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns one of \c llcas_lookup_result_t.
+ */
+LLCAS_PUBLIC llcas_lookup_result_t llcas_cas_contains_object(llcas_cas_t,
+ llcas_objectid_t,
+ bool globally,
+ char **error);
+
+/**
+ * Loads the object that \c llcas_objectid_t points to.
+ *
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns one of \c llcas_lookup_result_t.
+ */
+LLCAS_PUBLIC llcas_lookup_result_t llcas_cas_load_object(
+ llcas_cas_t, llcas_objectid_t, llcas_loaded_object_t *, char **error);
+
+/**
+ * Like \c llcas_cas_load_object but loading happens via a callback function.
+ * Whether the call is asynchronous or not depends on the implementation.
+ *
+ * \param ctx_cb pointer to pass to the callback function.
+ *
+ * \param[out] cancel_tok optional pointer to receive a \c llcas_cancellable_t.
+ */
+LLCAS_PUBLIC void llcas_cas_load_object_async(llcas_cas_t, llcas_objectid_t,
+ void *ctx_cb,
+ llcas_cas_load_object_cb,
+ llcas_cancellable_t *cancel_tok);
+
+/**
+ * Stores the object with the provided data buffer and \c llcas_objectid_t
+ * references, and provides its associated \c llcas_objectid_t.
+ *
+ * \param refs pointer to array of \c llcas_objectid_t. Can be \c NULL if
+ * \p refs_count is 0.
+ * \param refs_count number of \c llcas_objectid_t objects in the array.
+ * \param p_id pointer to store the returned \c llcas_objectid_t object.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_store_object(llcas_cas_t, llcas_data_t,
+ const llcas_objectid_t *refs,
+ size_t refs_count,
+ llcas_objectid_t *p_id, char **error);
+
+/**
+ * Stores the data of a file and provides its associated \c llcas_objectid_t.
+ *
+ * An underlying implementation could perform optimizations that reduce I/O
+ * and disk space consumption.
+ *
+ * If there are any concurrent modifications to the file, the contents in the
+ * CAS may be corrupt.
+ *
+ * \param filepath path to the file.
+ * \param p_id pointer to store the returned \c llcas_objectid_t object.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_cas_store_from_filepath(llcas_cas_t,
+ const char *filepath,
+ llcas_objectid_t *p_id,
+ char **error);
+
+/**
+ * \returns the data buffer of the provided \c llcas_loaded_object_t. The buffer
+ * pointer must be 8-byte aligned and \c NULL terminated. The memory that the
+ * buffer points to is valid for the lifetime of the \c llcas_cas_t object.
+ */
+LLCAS_PUBLIC llcas_data_t llcas_loaded_object_get_data(llcas_cas_t,
+ llcas_loaded_object_t);
+
+/**
+ * \returns the references of the provided \c llcas_loaded_object_t.
+ */
+LLCAS_PUBLIC llcas_object_refs_t
+ llcas_loaded_object_get_refs(llcas_cas_t, llcas_loaded_object_t);
+
+/**
+ * \returns the number of references in the provided \c llcas_object_refs_t.
+ */
+LLCAS_PUBLIC size_t llcas_object_refs_get_count(llcas_cas_t,
+ llcas_object_refs_t);
+
+/**
+ * \returns the \c llcas_objectid_t of the reference at \p index. It is invalid
+ * to pass an index that is out of the range of references.
+ */
+LLCAS_PUBLIC llcas_objectid_t llcas_object_refs_get_id(llcas_cas_t,
+ llcas_object_refs_t,
+ size_t index);
+
+/**
+ * Exports the data of an object to a file path. It does not include any
+ * references of the object.
+ *
+ * An underlying implementation could perform optimizations that reduce I/O
+ * and disk space consumption.
+ *
+ * \param filepath the file path to write the data to.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool
+llcas_loaded_object_export_data_to_filepath(llcas_cas_t, llcas_loaded_object_t,
+ const char *filepath, char **error);
+
+/**
+ * Retrieves the \c llcas_objectid_t value associated with a \p key.
+ *
+ * \param p_value pointer to store the returned \c llcas_objectid_t object.
+ * \param globally if true it is a hint to the underlying implementation that
+ * the lookup is profitable to be done on a distributed caching level, not just
+ * locally. The implementation is free to ignore this flag.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns one of \c llcas_lookup_result_t.
+ */
+LLCAS_PUBLIC llcas_lookup_result_t llcas_actioncache_get_for_digest(
+ llcas_cas_t, llcas_digest_t key, llcas_objectid_t *p_value, bool globally,
+ char **error);
+
+/**
+ * Like \c llcas_actioncache_get_for_digest but result is provided to a callback
+ * function. Whether the call is asynchronous or not depends on the
+ * implementation.
+ *
+ * \param ctx_cb pointer to pass to the callback function.
+ *
+ * \param[out] cancel_tok optional pointer to receive a \c llcas_cancellable_t.
+ */
+LLCAS_PUBLIC void llcas_actioncache_get_for_digest_async(
+ llcas_cas_t, llcas_digest_t key, bool globally, void *ctx_cb,
+ llcas_actioncache_get_cb, llcas_cancellable_t *cancel_tok);
+
+/**
+ * Associates a \c llcas_objectid_t \p value with a \p key. It is invalid to set
+ * a
diff erent \p value to the same \p key.
+ *
+ * \param globally if true it is a hint to the underlying implementation that
+ * the association is profitable to be done on a distributed caching level, not
+ * just locally. The implementation is free to ignore this flag.
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_actioncache_put_for_digest(llcas_cas_t,
+ llcas_digest_t key,
+ llcas_objectid_t value,
+ bool globally, char **error);
+
+/**
+ * Like \c llcas_actioncache_put_for_digest but result is provided to a callback
+ * function. Whether the call is asynchronous or not depends on the
+ * implementation.
+ *
+ * \param ctx_cb pointer to pass to the callback function.
+ *
+ * \param[out] cancel_tok optional pointer to receive a \c llcas_cancellable_t.
+ */
+LLCAS_PUBLIC void llcas_actioncache_put_for_digest_async(
+ llcas_cas_t, llcas_digest_t key, llcas_objectid_t value, bool globally,
+ void *ctx_cb, llcas_actioncache_put_cb, llcas_cancellable_t *cancel_tok);
+
+/**
+ * Validate the action cache contents.
+ *
+ * \param error optional pointer to receive an error message if an error
+ * occurred. If set, the memory it points to needs to be released via
+ * \c llcas_string_dispose.
+ * \returns true if there was an error, false otherwise.
+ */
+LLCAS_PUBLIC bool llcas_actioncache_validate(llcas_cas_t, char **error);
+
+LLVM_C_EXTERN_C_END
+
+#endif /* LLVM_C_CAS_PLUGINAPI_FUNCTIONS_H */
diff --git a/llvm/include/llvm-c/CAS/PluginAPI_types.h b/llvm/include/llvm-c/CAS/PluginAPI_types.h
new file mode 100644
index 0000000000000..6f969837446a4
--- /dev/null
+++ b/llvm/include/llvm-c/CAS/PluginAPI_types.h
@@ -0,0 +1,119 @@
+/*===----------------------------------------------------------------------===*\
+|* *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *|
+|* Exceptions. *|
+|* See https://llvm.org/LICENSE.txt for license information. *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *|
+|* *|
+|*===----------------------------------------------------------------------===*|
+|* *|
+|* The types for the LLVM CAS plugin API. *|
+|* The API is experimental and subject to change. *|
+|* *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_C_CAS_PLUGINAPI_TYPES_H
+#define LLVM_C_CAS_PLUGINAPI_TYPES_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define LLCAS_VERSION_MAJOR 0
+#define LLCAS_VERSION_MINOR 1
+
+typedef struct llcas_cas_options_s *llcas_cas_options_t;
+typedef struct llcas_cas_s *llcas_cas_t;
+typedef struct llcas_cancellable_s *llcas_cancellable_t;
+
+/**
+ * Digest hash bytes.
+ */
+typedef struct {
+ const uint8_t *data;
+ size_t size;
+} llcas_digest_t;
+
+/**
+ * Data buffer for stored CAS objects.
+ */
+typedef struct {
+ const void *data;
+ size_t size;
+} llcas_data_t;
+
+/**
+ * Identifier for a CAS object.
+ */
+typedef struct {
+ uint64_t opaque;
+} llcas_objectid_t;
+
+/**
+ * A loaded CAS object.
+ */
+typedef struct {
+ uint64_t opaque;
+} llcas_loaded_object_t;
+
+/**
+ * Object references for a CAS object.
+ */
+typedef struct {
+ uint64_t opaque_b;
+ uint64_t opaque_e;
+} llcas_object_refs_t;
+
+/**
+ * Return values for a load operation.
+ */
+typedef enum {
+ /**
+ * The object was found.
+ */
+ LLCAS_LOOKUP_RESULT_SUCCESS = 0,
+
+ /**
+ * The object was not found.
+ */
+ LLCAS_LOOKUP_RESULT_NOTFOUND = 1,
+
+ /**
+ * An error occurred.
+ */
+ LLCAS_LOOKUP_RESULT_ERROR = 2,
+} llcas_lookup_result_t;
+
+/**
+ * Callback for \c llcas_cas_load_object_async.
+ *
+ * \param ctx pointer passed through from the \c llcas_cas_load_object_async
+ * call.
+ * \param error message if an error occurred. If set, the memory it points to
+ * needs to be released via \c llcas_string_dispose.
+ */
+typedef void (*llcas_cas_load_object_cb)(void *ctx, llcas_lookup_result_t,
+ llcas_loaded_object_t, char *error);
+
+/**
+ * Callback for \c llcas_actioncache_get_for_digest_async.
+ *
+ * \param ctx pointer passed through from the
+ * \c llcas_actioncache_get_for_digest_async call.
+ * \param error message if an error occurred. If set, the memory it points to
+ * needs to be released via \c llcas_string_dispose.
+ */
+typedef void (*llcas_actioncache_get_cb)(void *ctx, llcas_lookup_result_t,
+ llcas_objectid_t, char *error);
+
+/**
+ * Callback for \c llcas_actioncache_put_for_digest_async.
+ *
+ * \param ctx pointer passed through from the
+ * \c llcas_actioncache_put_for_digest_async call.
+ * \param error message if an error occurred. If set, the memory it points to
+ * needs to be released via \c llcas_string_dispose.
+ */
+typedef void (*llcas_actioncache_put_cb)(void *ctx, bool failed, char *error);
+
+#endif /* LLVM_C_CAS_PLUGINAPI_TYPES_H */
diff --git a/llvm/include/llvm/CAS/ObjectStore.h b/llvm/include/llvm/CAS/ObjectStore.h
index f688e6c016a30..eb61b5f0eb36f 100644
--- a/llvm/include/llvm/CAS/ObjectStore.h
+++ b/llvm/include/llvm/CAS/ObjectStore.h
@@ -30,6 +30,7 @@ namespace cas {
class ObjectStore;
class ObjectProxy;
+class ActionCache;
/// Content-addressable storage for objects.
///
@@ -364,6 +365,20 @@ LLVM_ABI bool isOnDiskCASEnabled();
LLVM_ABI Expected<std::unique_ptr<ObjectStore>>
createOnDiskCAS(const Twine &Path);
+/// Create \c ObjectStore and \c ActionCache instances backed by a plugin that
+/// implements the C API in \c "llvm-c/CAS/PluginAPI_functions.h".
+///
+/// \param PluginPath path of the dynamic library to load.
+/// \param OnDiskPath local path that the plugin should use for any on-disk
+/// resources/caches.
+/// \param PluginArgs name/value pairs passed to the plugin as custom options;
+/// they are opaque to the client.
+LLVM_ABI Expected<
+ std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+createPluginCASDatabases(
+ StringRef PluginPath, StringRef OnDiskPath,
+ ArrayRef<std::pair<std::string, std::string>> PluginArgs);
+
} // namespace cas
} // namespace llvm
diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt
index b17fa84558bab..05e08dfab844e 100644
--- a/llvm/lib/CAS/CMakeLists.txt
+++ b/llvm/lib/CAS/CMakeLists.txt
@@ -21,6 +21,7 @@ add_llvm_component_library(LLVMCAS
OnDiskGraphDB.cpp
OnDiskKeyValueDB.cpp
OnDiskTrieRawHashMap.cpp
+ PluginCAS.cpp
UnifiedOnDiskCache.cpp
ADDITIONAL_HEADER_DIRS
diff --git a/llvm/lib/CAS/PluginAPI.h b/llvm/lib/CAS/PluginAPI.h
new file mode 100644
index 0000000000000..3a694ddca9ec4
--- /dev/null
+++ b/llvm/lib/CAS/PluginAPI.h
@@ -0,0 +1,129 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines \c llcas_functions_t, the table of function pointers that is
+/// populated by looking up the \c llcas_* symbols of a loaded CAS plugin.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CAS_PLUGINAPI_H
+#define LLVM_LIB_CAS_PLUGINAPI_H
+
+#include "llvm-c/CAS/PluginAPI_types.h"
+
+/// See documentation in \c "llvm-c/CAS/PluginAPI_functions.h" for how these
+/// functions are used.
+struct llcas_functions_t {
+ void (*get_plugin_version)(unsigned *major, unsigned *minor);
+
+ void (*string_dispose)(char *);
+
+ void (*cancellable_cancel)(llcas_cancellable_t);
+
+ void (*cancellable_dispose)(llcas_cancellable_t);
+
+ llcas_cas_options_t (*cas_options_create)(void);
+
+ void (*cas_options_dispose)(llcas_cas_options_t);
+
+ void (*cas_options_set_client_version)(llcas_cas_options_t, unsigned major,
+ unsigned minor);
+
+ void (*cas_options_set_ondisk_path)(llcas_cas_options_t, const char *path);
+
+ bool (*cas_options_set_option)(llcas_cas_options_t, const char *name,
+ const char *value, char **error);
+
+ llcas_cas_t (*cas_create)(llcas_cas_options_t, char **error);
+
+ void (*cas_dispose)(llcas_cas_t);
+
+ int64_t (*cas_get_ondisk_size)(llcas_cas_t, char **error);
+
+ bool (*cas_set_ondisk_size_limit)(llcas_cas_t, int64_t size_limit,
+ char **error);
+
+ bool (*cas_prune_ondisk_data)(llcas_cas_t, char **error);
+
+ bool (*cas_validate)(llcas_cas_t, bool check_hash, char **error);
+
+ unsigned (*digest_parse)(llcas_cas_t, const char *printed_digest,
+ uint8_t *bytes, size_t bytes_size, char **error);
+
+ bool (*digest_print)(llcas_cas_t, llcas_digest_t, char **printed_id,
+ char **error);
+
+ char *(*cas_get_hash_schema_name)(llcas_cas_t);
+
+ bool (*cas_get_objectid)(llcas_cas_t, llcas_digest_t, llcas_objectid_t *,
+ char **error);
+
+ llcas_digest_t (*objectid_get_digest)(llcas_cas_t, llcas_objectid_t);
+
+ llcas_lookup_result_t (*cas_contains_object)(llcas_cas_t, llcas_objectid_t,
+ bool globally, char **error);
+
+ llcas_lookup_result_t (*cas_load_object)(llcas_cas_t, llcas_objectid_t,
+ llcas_loaded_object_t *,
+ char **error);
+ void (*cas_load_object_async)(llcas_cas_t, llcas_objectid_t, void *ctx_cb,
+ llcas_cas_load_object_cb,
+ llcas_cancellable_t *);
+
+ bool (*cas_store_object)(llcas_cas_t, llcas_data_t,
+ const llcas_objectid_t *refs, size_t refs_count,
+ llcas_objectid_t *, char **error);
+
+ bool (*cas_store_from_filepath)(llcas_cas_t, const char *filepath,
+ llcas_objectid_t *, char **error);
+
+ llcas_data_t (*loaded_object_get_data)(llcas_cas_t, llcas_loaded_object_t);
+
+ llcas_object_refs_t (*loaded_object_get_refs)(llcas_cas_t,
+ llcas_loaded_object_t);
+
+ size_t (*object_refs_get_count)(llcas_cas_t, llcas_object_refs_t);
+
+ llcas_objectid_t (*object_refs_get_id)(llcas_cas_t, llcas_object_refs_t,
+ size_t index);
+
+ bool (*loaded_object_export_data_to_filepath)(llcas_cas_t,
+ llcas_loaded_object_t,
+ const char *filepath,
+ char **error);
+
+ /*===--------------------------------------------------------------------===*\
+ |* Action cache API
+ \*===--------------------------------------------------------------------===*/
+
+ llcas_lookup_result_t (*actioncache_get_for_digest)(llcas_cas_t,
+ llcas_digest_t key,
+ llcas_objectid_t *p_value,
+ bool globally,
+ char **error);
+
+ void (*actioncache_get_for_digest_async)(llcas_cas_t, llcas_digest_t key,
+ bool globally, void *ctx_cb,
+ llcas_actioncache_get_cb,
+ llcas_cancellable_t *);
+
+ bool (*actioncache_put_for_digest)(llcas_cas_t, llcas_digest_t key,
+ llcas_objectid_t value, bool globally,
+ char **error);
+
+ void (*actioncache_put_for_digest_async)(llcas_cas_t, llcas_digest_t key,
+ llcas_objectid_t value,
+ bool globally, void *ctx_cb,
+ llcas_actioncache_put_cb,
+ llcas_cancellable_t *);
+
+ bool (*actioncache_validate)(llcas_cas_t, char **error);
+};
+
+#endif // LLVM_LIB_CAS_PLUGINAPI_H
diff --git a/llvm/lib/CAS/PluginAPI_functions.def b/llvm/lib/CAS/PluginAPI_functions.def
new file mode 100644
index 0000000000000..996de2f4c7513
--- /dev/null
+++ b/llvm/lib/CAS/PluginAPI_functions.def
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// The list of functions that make up the LLVM CAS plugin API.
+///
+/// Format is Name/required. If 'required' is true then loading will fail if the
+/// symbol is missing, otherwise loading will continue and the function pointer
+/// will be null. Order is lexicographically by name.
+///
+//===----------------------------------------------------------------------===//
+
+CASPLUGINAPI_FUNCTION(actioncache_get_for_digest, true)
+CASPLUGINAPI_FUNCTION(actioncache_get_for_digest_async, true)
+CASPLUGINAPI_FUNCTION(actioncache_put_for_digest, true)
+CASPLUGINAPI_FUNCTION(actioncache_put_for_digest_async, true)
+CASPLUGINAPI_FUNCTION(actioncache_validate, false)
+CASPLUGINAPI_FUNCTION(cancellable_cancel, false)
+CASPLUGINAPI_FUNCTION(cancellable_dispose, false)
+CASPLUGINAPI_FUNCTION(cas_contains_object, true)
+CASPLUGINAPI_FUNCTION(cas_create, true)
+CASPLUGINAPI_FUNCTION(cas_dispose, true)
+CASPLUGINAPI_FUNCTION(cas_get_hash_schema_name, true)
+CASPLUGINAPI_FUNCTION(cas_get_objectid, true)
+CASPLUGINAPI_FUNCTION(cas_get_ondisk_size, false)
+CASPLUGINAPI_FUNCTION(cas_load_object, true)
+CASPLUGINAPI_FUNCTION(cas_load_object_async, true)
+CASPLUGINAPI_FUNCTION(cas_options_create, true)
+CASPLUGINAPI_FUNCTION(cas_options_dispose, true)
+CASPLUGINAPI_FUNCTION(cas_options_set_client_version, true)
+CASPLUGINAPI_FUNCTION(cas_options_set_ondisk_path, true)
+CASPLUGINAPI_FUNCTION(cas_options_set_option, true)
+CASPLUGINAPI_FUNCTION(cas_prune_ondisk_data, false)
+CASPLUGINAPI_FUNCTION(cas_set_ondisk_size_limit, false)
+CASPLUGINAPI_FUNCTION(cas_store_from_filepath, false)
+CASPLUGINAPI_FUNCTION(cas_store_object, true)
+CASPLUGINAPI_FUNCTION(cas_validate, false)
+CASPLUGINAPI_FUNCTION(digest_parse, true)
+CASPLUGINAPI_FUNCTION(digest_print, true)
+CASPLUGINAPI_FUNCTION(get_plugin_version, true)
+CASPLUGINAPI_FUNCTION(loaded_object_export_data_to_filepath, false)
+CASPLUGINAPI_FUNCTION(loaded_object_get_data, true)
+CASPLUGINAPI_FUNCTION(loaded_object_get_refs, true)
+CASPLUGINAPI_FUNCTION(object_refs_get_count, true)
+CASPLUGINAPI_FUNCTION(object_refs_get_id, true)
+CASPLUGINAPI_FUNCTION(objectid_get_digest, true)
+CASPLUGINAPI_FUNCTION(string_dispose, true)
diff --git a/llvm/lib/CAS/PluginCAS.cpp b/llvm/lib/CAS/PluginCAS.cpp
new file mode 100644
index 0000000000000..3b38a9d4fdb21
--- /dev/null
+++ b/llvm/lib/CAS/PluginCAS.cpp
@@ -0,0 +1,514 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements \c ObjectStore and \c ActionCache on top of a dynamically loaded
+/// plugin that provides the C API in \c "llvm-c/CAS/PluginAPI_functions.h".
+///
+/// The asynchronous entry points of the plugin API are not called yet; they
+/// will be wired up once \c ObjectStore and \c ActionCache grow asynchronous
+/// interfaces.
+///
+//===----------------------------------------------------------------------===//
+
+#include "PluginAPI.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+namespace {
+
+class PluginCASContext : public CASContext {
+public:
+ void printIDImpl(raw_ostream &OS, const CASID &ID) const final;
+
+ StringRef getHashSchemaIdentifier() const final { return SchemaName; }
+
+ static Expected<std::shared_ptr<PluginCASContext>>
+ create(StringRef PluginPath, StringRef OnDiskPath,
+ ArrayRef<std::pair<std::string, std::string>> PluginArgs);
+
+ ~PluginCASContext() { Functions.cas_dispose(c_cas); }
+
+ llcas_functions_t Functions{};
+ llcas_cas_t c_cas = nullptr;
+ std::string SchemaName;
+
+ static Error errorAndDispose(char *c_err, const llcas_functions_t &Funcs) {
+ Error E = createStringError(inconvertibleErrorCode(), c_err);
+ Funcs.string_dispose(c_err);
+ return E;
+ }
+
+ Error errorAndDispose(char *c_err) const {
+ return errorAndDispose(c_err, Functions);
+ }
+};
+
+} // anonymous namespace
+
+void PluginCASContext::printIDImpl(raw_ostream &OS, const CASID &ID) const {
+ ArrayRef<uint8_t> Hash = ID.getHash();
+ char *c_printed_id = nullptr;
+ char *c_err = nullptr;
+ if (Functions.digest_print(c_cas, llcas_digest_t{Hash.data(), Hash.size()},
+ &c_printed_id, &c_err))
+ report_fatal_error(errorAndDispose(c_err));
+ OS << c_printed_id;
+ Functions.string_dispose(c_printed_id);
+}
+
+Expected<std::shared_ptr<PluginCASContext>> PluginCASContext::create(
+ StringRef PluginPath, StringRef OnDiskPath,
+ ArrayRef<std::pair<std::string, std::string>> PluginArgs) {
+ auto reportError = [PluginPath](const Twine &Description) -> Error {
+ std::error_code EC = inconvertibleErrorCode();
+ return createStringError(EC, "error loading '" + PluginPath +
+ "': " + Description);
+ };
+
+ SmallString<256> PathBuf = PluginPath;
+ std::string ErrMsg;
+ sys::DynamicLibrary Lib =
+ sys::DynamicLibrary::getPermanentLibrary(PathBuf.c_str(), &ErrMsg);
+ if (!Lib.isValid())
+ return reportError(ErrMsg);
+
+ llcas_functions_t Functions{};
+
+#define CASPLUGINAPI_FUNCTION(name, required) \
+ if (!(Functions.name = (decltype(llcas_functions_t::name)) \
+ Lib.getAddressOfSymbol("llcas_" #name))) { \
+ if (required) \
+ return reportError("failed symbol 'llcas_" #name "' lookup"); \
+ }
+#include "PluginAPI_functions.def"
+#undef CASPLUGINAPI_FUNCTION
+
+ llcas_cas_options_t c_opts = Functions.cas_options_create();
+ scope_exit DisposeOptions([&]() { Functions.cas_options_dispose(c_opts); });
+
+ Functions.cas_options_set_client_version(c_opts, LLCAS_VERSION_MAJOR,
+ LLCAS_VERSION_MINOR);
+ SmallString<256> OnDiskPathBuf = OnDiskPath;
+ Functions.cas_options_set_ondisk_path(c_opts, OnDiskPathBuf.c_str());
+ for (const auto &Pair : PluginArgs) {
+ char *c_err = nullptr;
+ if (Functions.cas_options_set_option(c_opts, Pair.first.c_str(),
+ Pair.second.c_str(), &c_err))
+ return errorAndDispose(c_err, Functions);
+ }
+
+ char *c_err = nullptr;
+ llcas_cas_t c_cas = Functions.cas_create(c_opts, &c_err);
+ if (!c_cas)
+ return errorAndDispose(c_err, Functions);
+
+ char *c_schema = Functions.cas_get_hash_schema_name(c_cas);
+ std::string SchemaName = c_schema;
+ Functions.string_dispose(c_schema);
+
+ auto Ctx = std::make_shared<PluginCASContext>();
+ Ctx->Functions = Functions;
+ Ctx->c_cas = c_cas;
+ Ctx->SchemaName = std::move(SchemaName);
+ return Ctx;
+}
+
+//===----------------------------------------------------------------------===//
+// ObjectStore API
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class PluginObjectStore : public ObjectStore {
+public:
+ Expected<CASID> parseID(StringRef ID) final;
+ Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
+ ArrayRef<char> Data) final;
+ Expected<ObjectRef> storeFromFile(StringRef Path) final;
+ Error exportDataToFile(ObjectHandle Node, StringRef Path) const final;
+ CASID getID(ObjectRef Ref) const final;
+ std::optional<ObjectRef> getReference(const CASID &ID) const final;
+ Expected<bool> isMaterialized(ObjectRef Ref) const final;
+ Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) final;
+ uint64_t getDataSize(ObjectHandle Node) const final;
+ Error forEachRef(ObjectHandle Node,
+ function_ref<Error(ObjectRef)> Callback) const final;
+ ObjectRef readRef(ObjectHandle Node, size_t I) const final;
+ size_t getNumRefs(ObjectHandle Node) const final;
+ ArrayRef<char> getData(ObjectHandle Node,
+ bool RequiresNullTerminator = false) const final;
+ Error validateObject(const CASID &ID) final {
+ // Not supported yet. Always return success.
+ return Error::success();
+ }
+
+ Error validate(bool CheckHash) const final;
+
+ Error setSizeLimit(std::optional<uint64_t> SizeLimit) final;
+ Expected<std::optional<uint64_t>> getStorageSize() const final;
+ Error pruneStorageData() final;
+
+ PluginObjectStore(std::shared_ptr<PluginCASContext>);
+
+ /// Exposes \c makeObjectRef to the file-local helpers below.
+ ObjectRef makeRef(uint64_t InternalRef) const {
+ return makeObjectRef(InternalRef);
+ }
+
+ std::shared_ptr<PluginCASContext> Ctx;
+};
+
+} // anonymous namespace
+
+Expected<CASID> PluginObjectStore::parseID(StringRef ID) {
+ // Use big enough stack so that we don't have to allocate in the heap.
+ SmallString<148> IDBuf(ID);
+ SmallVector<uint8_t, 68> BytesBuf(68);
+
+ auto parseDigest = [&]() -> Expected<unsigned> {
+ char *c_err = nullptr;
+ unsigned NumBytes = Ctx->Functions.digest_parse(
+ Ctx->c_cas, IDBuf.c_str(), BytesBuf.data(), BytesBuf.size(), &c_err);
+ if (NumBytes == 0)
+ return Ctx->errorAndDispose(c_err);
+ return NumBytes;
+ };
+
+ Expected<unsigned> NumBytes = parseDigest();
+ if (!NumBytes)
+ return NumBytes.takeError();
+
+ if (*NumBytes > BytesBuf.size()) {
+ BytesBuf.resize(*NumBytes);
+ NumBytes = parseDigest();
+ if (!NumBytes)
+ return NumBytes.takeError();
+ assert(*NumBytes == BytesBuf.size());
+ } else {
+ BytesBuf.truncate(*NumBytes);
+ }
+
+ return CASID::create(Ctx.get(), toStringRef(BytesBuf));
+}
+
+Expected<ObjectRef> PluginObjectStore::store(ArrayRef<ObjectRef> Refs,
+ ArrayRef<char> Data) {
+ SmallVector<llcas_objectid_t, 64> c_ids;
+ c_ids.reserve(Refs.size());
+ for (ObjectRef Ref : Refs) {
+ c_ids.push_back(llcas_objectid_t{Ref.getInternalRef(*this)});
+ }
+
+ llcas_objectid_t c_stored_id;
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_store_object(
+ Ctx->c_cas, llcas_data_t{Data.data(), Data.size()}, c_ids.data(),
+ c_ids.size(), &c_stored_id, &c_err))
+ return Ctx->errorAndDispose(c_err);
+
+ return makeObjectRef(c_stored_id.opaque);
+}
+
+Expected<ObjectRef> PluginObjectStore::storeFromFile(StringRef Path) {
+ if (!Ctx->Functions.cas_store_from_filepath)
+ return ObjectStore::storeFromFile(Path);
+
+ llcas_objectid_t c_stored_id;
+ char *c_err = nullptr;
+ std::string PathStr = Path.str();
+ if (Ctx->Functions.cas_store_from_filepath(Ctx->c_cas, PathStr.c_str(),
+ &c_stored_id, &c_err))
+ return Ctx->errorAndDispose(c_err);
+
+ return makeObjectRef(c_stored_id.opaque);
+}
+
+Error PluginObjectStore::exportDataToFile(ObjectHandle Node,
+ StringRef Path) const {
+ if (!Ctx->Functions.loaded_object_export_data_to_filepath)
+ return ObjectStore::exportDataToFile(Node, Path);
+
+ char *c_err = nullptr;
+ std::string PathStr = Path.str();
+ if (Ctx->Functions.loaded_object_export_data_to_filepath(
+ Ctx->c_cas, llcas_loaded_object_t{Node.getInternalRef(*this)},
+ PathStr.c_str(), &c_err))
+ return Ctx->errorAndDispose(c_err);
+
+ return Error::success();
+}
+
+static StringRef toStringRef(llcas_digest_t c_digest) {
+ return StringRef((const char *)c_digest.data, c_digest.size);
+}
+
+CASID PluginObjectStore::getID(ObjectRef Ref) const {
+ llcas_objectid_t c_id{Ref.getInternalRef(*this)};
+ llcas_digest_t c_digest =
+ Ctx->Functions.objectid_get_digest(Ctx->c_cas, c_id);
+ return CASID::create(Ctx.get(), toStringRef(c_digest));
+}
+
+std::optional<ObjectRef>
+PluginObjectStore::getReference(const CASID &ID) const {
+ ArrayRef<uint8_t> Hash = ID.getHash();
+ llcas_objectid_t c_id;
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_get_objectid(
+ Ctx->c_cas, llcas_digest_t{Hash.data(), Hash.size()}, &c_id, &c_err))
+ report_fatal_error(Ctx->errorAndDispose(c_err));
+
+ return makeObjectRef(c_id.opaque);
+}
+
+Expected<bool> PluginObjectStore::isMaterialized(ObjectRef Ref) const {
+ llcas_objectid_t c_id{Ref.getInternalRef(*this)};
+ char *c_err = nullptr;
+ llcas_lookup_result_t c_result = Ctx->Functions.cas_contains_object(
+ Ctx->c_cas, c_id, /*globally=*/false, &c_err);
+ switch (c_result) {
+ case LLCAS_LOOKUP_RESULT_SUCCESS:
+ return true;
+ case LLCAS_LOOKUP_RESULT_NOTFOUND:
+ return false;
+ case LLCAS_LOOKUP_RESULT_ERROR:
+ return Ctx->errorAndDispose(c_err);
+ }
+ llvm_unreachable("unknown llcas_lookup_result_t value");
+}
+
+Expected<std::optional<ObjectHandle>>
+PluginObjectStore::loadIfExists(ObjectRef Ref) {
+ llcas_objectid_t c_id{Ref.getInternalRef(*this)};
+ llcas_loaded_object_t c_obj;
+ char *c_err = nullptr;
+ llcas_lookup_result_t c_result =
+ Ctx->Functions.cas_load_object(Ctx->c_cas, c_id, &c_obj, &c_err);
+ switch (c_result) {
+ case LLCAS_LOOKUP_RESULT_SUCCESS:
+ return makeObjectHandle(c_obj.opaque);
+ case LLCAS_LOOKUP_RESULT_NOTFOUND:
+ return std::nullopt;
+ case LLCAS_LOOKUP_RESULT_ERROR:
+ return Ctx->errorAndDispose(c_err);
+ }
+ llvm_unreachable("unknown llcas_lookup_result_t value");
+}
+
+namespace {
+
+class ObjectRefsWrapper {
+public:
+ ObjectRefsWrapper(const ObjectHandle &Node, const PluginObjectStore &Store)
+ : Store(Store), Ctx(*Store.Ctx) {
+ llcas_loaded_object_t c_obj{Node.getInternalRef(Store)};
+ this->c_refs = Ctx.Functions.loaded_object_get_refs(Ctx.c_cas, c_obj);
+ }
+
+ size_t size() const {
+ return Ctx.Functions.object_refs_get_count(Ctx.c_cas, c_refs);
+ }
+
+ ObjectRef operator[](size_t I) const {
+ llcas_objectid_t c_id =
+ Ctx.Functions.object_refs_get_id(Ctx.c_cas, c_refs, I);
+ return Store.makeRef(c_id.opaque);
+ }
+
+private:
+ const PluginObjectStore &Store;
+ PluginCASContext &Ctx;
+ llcas_object_refs_t c_refs;
+};
+
+} // namespace
+
+// FIXME: Replace forEachRef/readRef/getNumRefs APIs with an iterator interface.
+Error PluginObjectStore::forEachRef(
+ ObjectHandle Node, function_ref<Error(ObjectRef)> Callback) const {
+ ObjectRefsWrapper Refs(Node, *this);
+ for (unsigned I = 0, E = Refs.size(); I != E; ++I) {
+ if (Error E = Callback(Refs[I]))
+ return E;
+ }
+ return Error::success();
+}
+
+ObjectRef PluginObjectStore::readRef(ObjectHandle Node, size_t I) const {
+ ObjectRefsWrapper Refs(Node, *this);
+ return Refs[I];
+}
+
+size_t PluginObjectStore::getNumRefs(ObjectHandle Node) const {
+ ObjectRefsWrapper Refs(Node, *this);
+ return Refs.size();
+}
+
+// FIXME: Remove getDataSize(ObjectHandle) from API requirement,
+// \c getData(ObjectHandle) should be enough.
+uint64_t PluginObjectStore::getDataSize(ObjectHandle Node) const {
+ ArrayRef<char> Data = getData(Node);
+ return Data.size();
+}
+
+ArrayRef<char> PluginObjectStore::getData(ObjectHandle Node,
+ bool RequiresNullTerminator) const {
+ // FIXME: Remove RequiresNullTerminator from ObjectStore API requirement?
+ // It is a requirement for the plugin API.
+ llcas_data_t c_data = Ctx->Functions.loaded_object_get_data(
+ Ctx->c_cas, llcas_loaded_object_t{Node.getInternalRef(*this)});
+ return ArrayRef((const char *)c_data.data, c_data.size);
+}
+
+Error PluginObjectStore::setSizeLimit(std::optional<uint64_t> SizeLimit) {
+ if (Ctx->Functions.cas_set_ondisk_size_limit) {
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_set_ondisk_size_limit(Ctx->c_cas,
+ SizeLimit.value_or(0), &c_err))
+ return Ctx->errorAndDispose(c_err);
+ }
+ return Error::success();
+}
+
+Expected<std::optional<uint64_t>> PluginObjectStore::getStorageSize() const {
+ if (!Ctx->Functions.cas_get_ondisk_size)
+ return std::nullopt;
+ char *c_err = nullptr;
+ int64_t ret = Ctx->Functions.cas_get_ondisk_size(Ctx->c_cas, &c_err);
+ switch (ret) {
+ case -1:
+ return std::nullopt;
+ case -2:
+ return Ctx->errorAndDispose(c_err);
+ default:
+ return ret;
+ }
+}
+
+Error PluginObjectStore::pruneStorageData() {
+ if (Ctx->Functions.cas_prune_ondisk_data) {
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_prune_ondisk_data(Ctx->c_cas, &c_err))
+ return Ctx->errorAndDispose(c_err);
+ }
+ return Error::success();
+}
+
+Error PluginObjectStore::validate(bool CheckHash) const {
+ if (Ctx->Functions.cas_validate) {
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_validate(Ctx->c_cas, CheckHash, &c_err))
+ return Ctx->errorAndDispose(c_err);
+ return Error::success();
+ }
+ return createStringError("plugin cas doesn't support validation");
+}
+
+PluginObjectStore::PluginObjectStore(std::shared_ptr<PluginCASContext> CASCtx)
+ : ObjectStore(*CASCtx), Ctx(std::move(CASCtx)) {}
+
+//===----------------------------------------------------------------------===//
+// ActionCache API
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class PluginActionCache : public ActionCache {
+public:
+ Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ResolvedKey,
+ bool CanBeDistributed) const final;
+
+ Error putImpl(ArrayRef<uint8_t> ResolvedKey, const CASID &Result,
+ bool CanBeDistributed) final;
+
+ PluginActionCache(std::shared_ptr<PluginCASContext>);
+
+ Error validate() const final;
+
+private:
+ std::shared_ptr<PluginCASContext> Ctx;
+};
+
+} // anonymous namespace
+
+Expected<std::optional<CASID>>
+PluginActionCache::getImpl(ArrayRef<uint8_t> ResolvedKey,
+ bool CanBeDistributed) const {
+ llcas_objectid_t c_value;
+ char *c_err = nullptr;
+ llcas_lookup_result_t c_result = Ctx->Functions.actioncache_get_for_digest(
+ Ctx->c_cas, llcas_digest_t{ResolvedKey.data(), ResolvedKey.size()},
+ &c_value, CanBeDistributed, &c_err);
+ switch (c_result) {
+ case LLCAS_LOOKUP_RESULT_SUCCESS: {
+ llcas_digest_t c_digest =
+ Ctx->Functions.objectid_get_digest(Ctx->c_cas, c_value);
+ return CASID::create(Ctx.get(), toStringRef(c_digest));
+ }
+ case LLCAS_LOOKUP_RESULT_NOTFOUND:
+ return std::nullopt;
+ case LLCAS_LOOKUP_RESULT_ERROR:
+ return Ctx->errorAndDispose(c_err);
+ }
+ llvm_unreachable("unknown llcas_lookup_result_t value");
+}
+
+Error PluginActionCache::putImpl(ArrayRef<uint8_t> ResolvedKey,
+ const CASID &Result, bool CanBeDistributed) {
+ ArrayRef<uint8_t> Hash = Result.getHash();
+ llcas_objectid_t c_value;
+ char *c_err = nullptr;
+ if (Ctx->Functions.cas_get_objectid(Ctx->c_cas,
+ llcas_digest_t{Hash.data(), Hash.size()},
+ &c_value, &c_err))
+ return Ctx->errorAndDispose(c_err);
+
+ if (Ctx->Functions.actioncache_put_for_digest(
+ Ctx->c_cas, llcas_digest_t{ResolvedKey.data(), ResolvedKey.size()},
+ c_value, CanBeDistributed, &c_err))
+ return Ctx->errorAndDispose(c_err);
+
+ return Error::success();
+}
+
+PluginActionCache::PluginActionCache(std::shared_ptr<PluginCASContext> CASCtx)
+ : ActionCache(*CASCtx), Ctx(std::move(CASCtx)) {}
+
+Error PluginActionCache::validate() const {
+ if (Ctx->Functions.actioncache_validate) {
+ char *c_err = nullptr;
+ if (Ctx->Functions.actioncache_validate(Ctx->c_cas, &c_err))
+ return Ctx->errorAndDispose(c_err);
+ return Error::success();
+ }
+ return createStringError("plugin action cache doesn't support validation");
+}
+
+//===----------------------------------------------------------------------===//
+// createPluginCASDatabases API
+//===----------------------------------------------------------------------===//
+
+Expected<std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+cas::createPluginCASDatabases(
+ StringRef PluginPath, StringRef OnDiskPath,
+ ArrayRef<std::pair<std::string, std::string>> PluginArgs) {
+ std::shared_ptr<PluginCASContext> Ctx;
+ if (Error E = PluginCASContext::create(PluginPath, OnDiskPath, PluginArgs)
+ .moveInto(Ctx))
+ return std::move(E);
+ auto CAS = std::make_shared<PluginObjectStore>(Ctx);
+ auto AC = std::make_shared<PluginActionCache>(std::move(Ctx));
+ return std::make_pair(std::move(CAS), std::move(AC));
+}
diff --git a/llvm/tools/libCASPluginTest/CMakeLists.txt b/llvm/tools/libCASPluginTest/CMakeLists.txt
new file mode 100644
index 0000000000000..3e9723a23d516
--- /dev/null
+++ b/llvm/tools/libCASPluginTest/CMakeLists.txt
@@ -0,0 +1,18 @@
+if (NOT LLVM_ENABLE_ONDISK_CAS)
+ # The plugin is implemented on top of UnifiedOnDiskCache, which is only
+ # functional when the on-disk CAS is enabled.
+ return()
+endif()
+
+set(LLVM_LINK_COMPONENTS
+ CAS
+ Support
+ )
+
+set(SOURCES
+ libCASPluginTest.cpp
+ )
+
+set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libCASPluginTest.exports)
+
+add_llvm_library(CASPluginTest SHARED ${SOURCES})
diff --git a/llvm/tools/libCASPluginTest/libCASPluginTest.cpp b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp
new file mode 100644
index 0000000000000..04402bfdaca0f
--- /dev/null
+++ b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp
@@ -0,0 +1,779 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implementation of the LLVM CAS plugin API, for testing purposes.
+///
+/// It is backed by \c UnifiedOnDiskCache and can optionally be given a second
+/// on-disk path via the \c upstream-path option, which it uses to simulate
+/// "uploading"/"downloading" objects to/from a distributed CAS.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm-c/CAS/PluginAPI_functions.h"
+#include "llvm/CAS/BuiltinObjectHasher.h"
+#include "llvm/CAS/CASID.h"
+#include "llvm/CAS/OnDiskKeyValueDB.h"
+#include "llvm/CAS/UnifiedOnDiskCache.h"
+#include "llvm/Support/CBindingWrapping.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SHA1.h"
+#include "llvm/Support/ThreadPool.h"
+#include <mutex>
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::ondisk;
+
+namespace llvm::cas::ondisk {
+/// Declared in the private "OnDiskCommon.h"; see \c setSmallMaxMappingSize.
+void setMaxMappingSize(uint64_t Size);
+} // namespace llvm::cas::ondisk
+
+/// This plugin exists only for testing, and a test process can create many
+/// instances of it. Keep the on-disk mappings small so that they stay cheap;
+/// the default sizes are measured in gigabytes per instance.
+///
+/// This has to happen inside the plugin: it links its own copy of LLVMCAS, so
+/// the setting the test binary applies to itself does not reach us.
+static void setSmallMaxMappingSize() {
+ static std::once_flag Flag;
+ std::call_once(Flag, [] { setMaxMappingSize(100 * 1024 * 1024); });
+}
+
+static char *copyNewMallocString(StringRef Str) {
+ char *c_str = (char *)malloc(Str.size() + 1);
+ std::uninitialized_copy(Str.begin(), Str.end(), c_str);
+ c_str[Str.size()] = '\0';
+ return c_str;
+}
+
+template <typename ResT>
+static ResT reportError(Error &&E, char **error, ResT Result = ResT()) {
+ if (error)
+ *error = copyNewMallocString(toString(std::move(E)));
+ return Result;
+}
+
+void llcas_get_plugin_version(unsigned *major, unsigned *minor) {
+ *major = LLCAS_VERSION_MAJOR;
+ *minor = LLCAS_VERSION_MINOR;
+}
+
+void llcas_string_dispose(char *str) { free(str); }
+
+namespace {
+
+struct CancellableState {
+ std::atomic<bool> Cancelled{false};
+};
+
+struct CancellableWrap {
+ std::shared_ptr<CancellableState> State;
+};
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CancellableWrap, llcas_cancellable_t)
+
+} // namespace
+
+void llcas_cancellable_cancel(llcas_cancellable_t c_cancellable) {
+ unwrap(c_cancellable)->State->Cancelled = true;
+}
+
+void llcas_cancellable_dispose(llcas_cancellable_t c_cancellable) {
+ delete unwrap(c_cancellable);
+}
+
+namespace {
+
+struct CASPluginOptions {
+ std::string OnDiskPath;
+ std::string UpstreamPath;
+ std::string FirstPrefix;
+ std::string SecondPrefix;
+ bool SimulateMissingObjects = false;
+ bool Logging = true;
+
+ Error setOption(StringRef Name, StringRef Value);
+};
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CASPluginOptions, llcas_cas_options_t)
+
+} // namespace
+
+Error CASPluginOptions::setOption(StringRef Name, StringRef Value) {
+ if (Name == "first-prefix")
+ FirstPrefix = Value;
+ else if (Name == "second-prefix")
+ SecondPrefix = Value;
+ else if (Name == "upstream-path")
+ UpstreamPath = Value;
+ else if (Name == "simulate-missing-objects")
+ SimulateMissingObjects = true;
+ else if (Name == "no-logging")
+ Logging = false;
+ else
+ return createStringError(errc::invalid_argument,
+ Twine("unknown option: ") + Name);
+ return Error::success();
+}
+
+llcas_cas_options_t llcas_cas_options_create(void) {
+ return wrap(new CASPluginOptions());
+}
+
+void llcas_cas_options_dispose(llcas_cas_options_t c_opts) {
+ delete unwrap(c_opts);
+}
+
+void llcas_cas_options_set_ondisk_path(llcas_cas_options_t c_opts,
+ const char *path) {
+ auto &Opts = *unwrap(c_opts);
+ Opts.OnDiskPath = path;
+}
+
+bool llcas_cas_options_set_option(llcas_cas_options_t c_opts, const char *name,
+ const char *value, char **error) {
+ auto &Opts = *unwrap(c_opts);
+ if (Error E = Opts.setOption(name, value))
+ return reportError(std::move(E), error, true);
+ return false;
+}
+
+namespace {
+
+using HasherT = SHA1;
+using HashType = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
+
+class PluginCASContext : public CASContext {
+ void printIDImpl(raw_ostream &OS, const CASID &ID) const final {
+ PluginCASContext::printID(ID.getHash(), OS);
+ }
+
+public:
+ static StringRef getHashName() { return "SHA1"; }
+ StringRef getHashSchemaIdentifier() const final {
+ static const std::string ID =
+ ("llvm.cas.builtin.v2[" + getHashName() + "]").str();
+ return ID;
+ }
+
+ PluginCASContext() = default;
+
+ static Expected<HashType> parseID(StringRef Reference) {
+ if (!Reference.consume_front("llvmcas://"))
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ "invalid cas-id '" + Reference + "'");
+
+ if (Reference.size() != 2 * sizeof(HashType))
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ "wrong size for cas-id hash '" + Reference + "'");
+
+ std::string Binary;
+ if (!tryGetFromHex(Reference, Binary))
+ return createStringError(
+ std::make_error_code(std::errc::invalid_argument),
+ "invalid hash in cas-id '" + Reference + "'");
+
+ assert(Binary.size() == sizeof(HashType));
+ HashType Digest;
+ llvm::copy(Binary, Digest.data());
+ return Digest;
+ }
+
+ static void printID(ArrayRef<uint8_t> Digest, raw_ostream &OS) {
+ SmallString<64> Hash;
+ toHex(Digest, /*LowerCase=*/true, Hash);
+ OS << "llvmcas://" << Hash;
+ }
+};
+
+struct CASWrapper {
+ std::string FirstPrefix;
+ std::string SecondPrefix;
+ /// If true, asynchronous "download" of an object will treat it as missing.
+ bool SimulateMissingObjects = false;
+ bool Logging = true;
+ std::unique_ptr<UnifiedOnDiskCache> DB;
+ /// Used for testing the \c globally parameter of action cache APIs. Simulates
+ /// "uploading"/"downloading" objects from/to the primary on-disk path.
+ std::unique_ptr<UnifiedOnDiskCache> UpstreamDB;
+ DefaultThreadPool Pool{llvm::hardware_concurrency()};
+
+ std::mutex Lock{};
+
+ /// Check if the object is contained, in the "local" CAS only or "globally".
+ bool containsObject(ObjectID ID, bool Globally);
+
+ /// Load the object, potentially "downloading" it from upstream.
+ Expected<std::optional<ondisk::ObjectHandle>> loadObject(ObjectID ID);
+
+ /// "Uploads" a key and the associated full node graph.
+ Error upstreamKey(ArrayRef<uint8_t> Key, ObjectID Value);
+
+ /// "Downloads" the ID associated with the key but not the node data. The node
+ /// itself and the rest of the nodes in the graph will be "downloaded" lazily
+ /// as they are visited.
+ Expected<std::optional<ObjectID>> downstreamKey(ArrayRef<uint8_t> Key);
+
+ /// Synchronized access to \c llvm::errs().
+ void syncErrs(llvm::function_ref<void(raw_ostream &OS)> Fn) {
+ if (!Logging) {
+ // Ignore log output.
+ SmallString<32> Buf;
+ raw_svector_ostream OS(Buf);
+ Fn(OS);
+ return;
+ }
+ std::unique_lock<std::mutex> LockGuard(Lock);
+ Fn(errs());
+ errs().flush();
+ }
+
+private:
+ /// "Uploads" the full object node graph.
+ Expected<ObjectID> upstreamNode(ObjectID Node);
+ /// "Downloads" only a single object node. The rest of the nodes in the graph
+ /// will be "downloaded" lazily as they are visited.
+ Expected<ObjectID> downstreamNode(ObjectID Node);
+};
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CASWrapper, llcas_cas_t)
+
+} // namespace
+
+bool CASWrapper::containsObject(ObjectID ID, bool Globally) {
+ if (DB->getGraphDB().containsObject(ID))
+ return true;
+ if (!Globally || !UpstreamDB)
+ return false;
+
+ auto UpstreamID = expectedToOptional(
+ UpstreamDB->getGraphDB().getReference(DB->getGraphDB().getDigest(ID)));
+
+ if (!UpstreamID)
+ return false;
+
+ return UpstreamDB->getGraphDB().containsObject(*UpstreamID);
+}
+
+Expected<std::optional<ondisk::ObjectHandle>>
+CASWrapper::loadObject(ObjectID ID) {
+ std::optional<ondisk::ObjectHandle> Obj;
+ if (Error E = DB->getGraphDB().load(ID).moveInto(Obj))
+ return std::move(E);
+ if (Obj)
+ return Obj;
+ if (!UpstreamDB)
+ return std::nullopt;
+
+ // Try "downloading" the node from upstream.
+ auto UpstreamID =
+ UpstreamDB->getGraphDB().getReference(DB->getGraphDB().getDigest(ID));
+ if (!UpstreamID)
+ return UpstreamID.takeError();
+ std::optional<ObjectID> Ret;
+ if (Error E = downstreamNode(*UpstreamID).moveInto(Ret))
+ return std::move(E);
+ return DB->getGraphDB().load(ID);
+}
+
+/// Imports a single object node.
+static Expected<ObjectID> importNode(ObjectID FromID, OnDiskGraphDB &FromDB,
+ OnDiskGraphDB &ToDB) {
+ auto ToID = ToDB.getReference(FromDB.getDigest(FromID));
+ if (!ToID)
+ return ToID.takeError();
+ if (ToDB.containsObject(*ToID))
+ return ToID;
+
+ std::optional<ondisk::ObjectHandle> FromH;
+ if (Error E = FromDB.load(FromID).moveInto(FromH))
+ return std::move(E);
+ if (!FromH)
+ return ToID;
+
+ auto Data = FromDB.getObjectData(*FromH);
+ auto FromRefs = FromDB.getObjectRefs(*FromH);
+ SmallVector<ObjectID> Refs;
+ for (ObjectID FromRef : FromRefs) {
+ auto Ref = ToDB.getReference(FromDB.getDigest(FromRef));
+ if (!Ref)
+ return Ref.takeError();
+ Refs.push_back(*Ref);
+ }
+
+ if (Error E = ToDB.store(*ToID, Refs, Data))
+ return std::move(E);
+ return ToID;
+}
+
+Expected<ObjectID> CASWrapper::upstreamNode(ObjectID Node) {
+ OnDiskGraphDB &FromDB = DB->getGraphDB();
+ OnDiskGraphDB &ToDB = UpstreamDB->getGraphDB();
+
+ std::optional<ondisk::ObjectHandle> FromH;
+ if (Error E = FromDB.load(Node).moveInto(FromH))
+ return std::move(E);
+ if (!FromH)
+ return createStringError(errc::invalid_argument, "node doesn't exist");
+
+ for (ObjectID Ref : FromDB.getObjectRefs(*FromH)) {
+ std::optional<ObjectID> ID;
+ if (Error E = upstreamNode(Ref).moveInto(ID))
+ return std::move(E);
+ }
+
+ return importNode(Node, FromDB, ToDB);
+}
+
+Expected<ObjectID> CASWrapper::downstreamNode(ObjectID Node) {
+ OnDiskGraphDB &FromDB = UpstreamDB->getGraphDB();
+ OnDiskGraphDB &ToDB = DB->getGraphDB();
+ return importNode(Node, FromDB, ToDB);
+}
+
+static Expected<ObjectID> cachePut(OnDiskKeyValueDB &DB, ArrayRef<uint8_t> Key,
+ ObjectID ID) {
+ auto Value = UnifiedOnDiskCache::getValueFromObjectID(ID);
+ auto Result = DB.put(Key, Value);
+ if (!Result)
+ return Result.takeError();
+ return UnifiedOnDiskCache::getObjectIDFromValue(*Result);
+}
+
+static Expected<std::optional<ObjectID>> cacheGet(OnDiskKeyValueDB &DB,
+ ArrayRef<uint8_t> Key) {
+ auto Result = DB.get(Key);
+ if (!Result)
+ return Result.takeError();
+ if (!*Result)
+ return std::nullopt;
+ return UnifiedOnDiskCache::getObjectIDFromValue(**Result);
+}
+
+Error CASWrapper::upstreamKey(ArrayRef<uint8_t> Key, ObjectID Value) {
+ if (!UpstreamDB)
+ return Error::success();
+ Expected<ObjectID> UpstreamVal = upstreamNode(Value);
+ if (!UpstreamVal)
+ return UpstreamVal.takeError();
+ Expected<ObjectID> PutValue =
+ cachePut(UpstreamDB->getKeyValueDB(), Key, *UpstreamVal);
+ if (!PutValue)
+ return PutValue.takeError();
+ assert(*PutValue == *UpstreamVal);
+ return Error::success();
+}
+
+Expected<std::optional<ObjectID>>
+CASWrapper::downstreamKey(ArrayRef<uint8_t> Key) {
+ if (!UpstreamDB)
+ return std::nullopt;
+ std::optional<ObjectID> UpstreamValue;
+ if (Error E =
+ cacheGet(UpstreamDB->getKeyValueDB(), Key).moveInto(UpstreamValue))
+ return std::move(E);
+ if (!UpstreamValue)
+ return std::nullopt;
+
+ auto Value = DB->getGraphDB().getReference(
+ UpstreamDB->getGraphDB().getDigest(*UpstreamValue));
+ if (!Value)
+ return Value.takeError();
+ Expected<ObjectID> PutValue = cachePut(DB->getKeyValueDB(), Key, *Value);
+ if (!PutValue)
+ return PutValue.takeError();
+ assert(*PutValue == *Value);
+ return PutValue;
+}
+
+llcas_cas_t llcas_cas_create(llcas_cas_options_t c_opts, char **error) {
+ auto &Opts = *unwrap(c_opts);
+ setSmallMaxMappingSize();
+ Expected<std::unique_ptr<UnifiedOnDiskCache>> DB = UnifiedOnDiskCache::open(
+ Opts.OnDiskPath, /*SizeLimit=*/std::nullopt,
+ PluginCASContext::getHashName(), sizeof(HashType));
+ if (!DB)
+ return reportError<llcas_cas_t>(DB.takeError(), error);
+
+ std::unique_ptr<UnifiedOnDiskCache> UpstreamDB;
+ if (!Opts.UpstreamPath.empty()) {
+ if (Error E = UnifiedOnDiskCache::open(
+ Opts.UpstreamPath, /*SizeLimit=*/std::nullopt,
+ PluginCASContext::getHashName(), sizeof(HashType))
+ .moveInto(UpstreamDB))
+ return reportError<llcas_cas_t>(std::move(E), error);
+ }
+
+ return wrap(new CASWrapper{Opts.FirstPrefix, Opts.SecondPrefix,
+ Opts.SimulateMissingObjects, Opts.Logging,
+ std::move(*DB), std::move(UpstreamDB)});
+}
+
+void llcas_cas_dispose(llcas_cas_t c_cas) { delete unwrap(c_cas); }
+
+int64_t llcas_cas_get_ondisk_size(llcas_cas_t c_cas, char **error) {
+ return unwrap(c_cas)->DB->getStorageSize();
+}
+
+bool llcas_cas_set_ondisk_size_limit(llcas_cas_t c_cas, int64_t size_limit,
+ char **error) {
+ std::optional<uint64_t> SizeLimit;
+ if (size_limit < 0) {
+ return reportError(
+ llvm::createStringError(
+ llvm::inconvertibleErrorCode(),
+ "invalid size limit passed to llcas_cas_set_ondisk_size_limit"),
+ error, true);
+ }
+ if (size_limit > 0) {
+ SizeLimit = size_limit;
+ }
+ unwrap(c_cas)->DB->setSizeLimit(SizeLimit);
+ return false;
+}
+
+bool llcas_cas_prune_ondisk_data(llcas_cas_t c_cas, char **error) {
+ if (Error E = unwrap(c_cas)->DB->collectGarbage())
+ return reportError(std::move(E), error, true);
+ return false;
+}
+
+void llcas_cas_options_set_client_version(llcas_cas_options_t, unsigned major,
+ unsigned minor) {
+ // Ignore for now.
+}
+
+char *llcas_cas_get_hash_schema_name(llcas_cas_t) {
+ // Using same name as builtin CAS so that it's interchangeable for testing
+ // purposes.
+ return copyNewMallocString("llvm.cas.builtin.v2[BLAKE3]");
+}
+
+unsigned llcas_digest_parse(llcas_cas_t c_cas, const char *printed_digest,
+ uint8_t *bytes, size_t bytes_size, char **error) {
+ auto &Wrapper = *unwrap(c_cas);
+ if (bytes_size < sizeof(HashType))
+ return sizeof(HashType);
+
+ StringRef PrintedDigest = printed_digest;
+ bool Consumed = PrintedDigest.consume_front(Wrapper.FirstPrefix);
+ assert(Consumed);
+ (void)Consumed;
+ Consumed = PrintedDigest.consume_front(Wrapper.SecondPrefix);
+ assert(Consumed);
+ (void)Consumed;
+
+ Expected<HashType> Digest = PluginCASContext::parseID(PrintedDigest);
+ if (!Digest)
+ return reportError(Digest.takeError(), error, 0);
+ std::uninitialized_copy(Digest->begin(), Digest->end(), bytes);
+ return Digest->size();
+}
+
+bool llcas_digest_print(llcas_cas_t c_cas, llcas_digest_t c_digest,
+ char **printed_id, char **error) {
+ auto &Wrapper = *unwrap(c_cas);
+ SmallString<74> PrintDigest;
+ raw_svector_ostream OS(PrintDigest);
+ // Include these for testing purposes.
+ OS << Wrapper.FirstPrefix << Wrapper.SecondPrefix;
+ PluginCASContext::printID(ArrayRef(c_digest.data, c_digest.size), OS);
+ *printed_id = copyNewMallocString(PrintDigest);
+ return false;
+}
+
+bool llcas_cas_get_objectid(llcas_cas_t c_cas, llcas_digest_t c_digest,
+ llcas_objectid_t *c_id_p, char **error) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ auto ID = CAS.getReference(ArrayRef(c_digest.data, c_digest.size));
+ if (!ID)
+ return reportError(ID.takeError(), error, true);
+
+ *c_id_p = llcas_objectid_t{ID->getOpaqueData()};
+ return false;
+}
+
+llcas_digest_t llcas_objectid_get_digest(llcas_cas_t c_cas,
+ llcas_objectid_t c_id) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ ObjectID ID = ObjectID::fromOpaqueData(c_id.opaque);
+ ArrayRef<uint8_t> Digest = CAS.getDigest(ID);
+ return llcas_digest_t{Digest.data(), Digest.size()};
+}
+
+llcas_lookup_result_t llcas_cas_contains_object(llcas_cas_t c_cas,
+ llcas_objectid_t c_id,
+ bool globally, char **error) {
+ ObjectID ID = ObjectID::fromOpaqueData(c_id.opaque);
+ return unwrap(c_cas)->containsObject(ID, globally)
+ ? LLCAS_LOOKUP_RESULT_SUCCESS
+ : LLCAS_LOOKUP_RESULT_NOTFOUND;
+}
+
+llcas_lookup_result_t llcas_cas_load_object(llcas_cas_t c_cas,
+ llcas_objectid_t c_id,
+ llcas_loaded_object_t *c_obj_p,
+ char **error) {
+ ObjectID ID = ObjectID::fromOpaqueData(c_id.opaque);
+ Expected<std::optional<ondisk::ObjectHandle>> ObjOpt =
+ unwrap(c_cas)->loadObject(ID);
+ if (!ObjOpt)
+ return reportError(ObjOpt.takeError(), error, LLCAS_LOOKUP_RESULT_ERROR);
+ if (!*ObjOpt)
+ return LLCAS_LOOKUP_RESULT_NOTFOUND;
+
+ ondisk::ObjectHandle Obj = **ObjOpt;
+ *c_obj_p = llcas_loaded_object_t{Obj.getOpaqueData()};
+ return LLCAS_LOOKUP_RESULT_SUCCESS;
+}
+
+void llcas_cas_load_object_async(llcas_cas_t c_cas, llcas_objectid_t c_id,
+ void *ctx_cb, llcas_cas_load_object_cb cb,
+ llcas_cancellable_t *c_cancellable) {
+ auto CancelState = std::make_shared<CancellableState>();
+ if (c_cancellable) {
+ *c_cancellable = wrap(new CancellableWrap{CancelState});
+ }
+
+ std::string PrintedDigest;
+ {
+ llcas_digest_t c_digest = llcas_objectid_get_digest(c_cas, c_id);
+ char *printed_id;
+ char *c_err;
+ bool failed = llcas_digest_print(c_cas, c_digest, &printed_id, &c_err);
+ if (failed)
+ report_fatal_error(Twine("digest printing failed: ") + c_err);
+ PrintedDigest = printed_id;
+ llcas_string_dispose(printed_id);
+ }
+
+ auto passObject = [ctx_cb,
+ cb](Expected<std::optional<ondisk::ObjectHandle>> Obj) {
+ if (!Obj) {
+ cb(ctx_cb, LLCAS_LOOKUP_RESULT_ERROR, llcas_loaded_object_t(),
+ copyNewMallocString(toString(Obj.takeError())));
+ } else if (!*Obj) {
+ cb(ctx_cb, LLCAS_LOOKUP_RESULT_NOTFOUND, llcas_loaded_object_t(),
+ nullptr);
+ } else {
+ cb(ctx_cb, LLCAS_LOOKUP_RESULT_SUCCESS,
+ llcas_loaded_object_t{(*Obj)->getOpaqueData()}, nullptr);
+ }
+ };
+
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ ObjectID ID = ObjectID::fromOpaqueData(c_id.opaque);
+ if (CAS.containsObject(ID)) {
+ unwrap(c_cas)->syncErrs([&](raw_ostream &OS) {
+ OS << "load_object_async existing: " << PrintedDigest << '\n';
+ });
+ return passObject(unwrap(c_cas)->loadObject(ID));
+ }
+
+ if (!unwrap(c_cas)->UpstreamDB)
+ return passObject(std::nullopt);
+
+ // Try "downloading" the node from upstream.
+
+ unwrap(c_cas)->syncErrs([&](raw_ostream &OS) {
+ OS << "load_object_async downstream begin: " << PrintedDigest << '\n';
+ });
+ unwrap(c_cas)->Pool.async([=] {
+ // Wait a bit for the caller to proceed.
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ auto &Wrap = *unwrap(c_cas);
+ if (CancelState->Cancelled) {
+ Wrap.syncErrs([&](raw_ostream &OS) {
+ OS << "load_object_async cancelled: " << PrintedDigest << '\n';
+ });
+ return passObject(std::nullopt);
+ }
+ Wrap.syncErrs([&](raw_ostream &OS) {
+ OS << "load_object_async downstream end: " << PrintedDigest << '\n';
+ });
+ if (Wrap.SimulateMissingObjects)
+ return passObject(std::nullopt);
+ passObject(Wrap.loadObject(ID));
+ });
+}
+
+bool llcas_cas_store_object(llcas_cas_t c_cas, llcas_data_t c_data,
+ const llcas_objectid_t *c_refs, size_t c_refs_count,
+ llcas_objectid_t *c_id_p, char **error) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ SmallVector<ObjectID, 64> Refs;
+ Refs.reserve(c_refs_count);
+ for (unsigned I = 0; I != c_refs_count; ++I) {
+ Refs.push_back(ObjectID::fromOpaqueData(c_refs[I].opaque));
+ }
+ ArrayRef Data((const char *)c_data.data, c_data.size);
+
+ SmallVector<ArrayRef<uint8_t>, 8> RefHashes;
+ RefHashes.reserve(c_refs_count);
+ for (ObjectID Ref : Refs)
+ RefHashes.push_back(CAS.getDigest(Ref));
+ HashType Digest = BuiltinObjectHasher<HasherT>::hashObject(RefHashes, Data);
+ auto StoredID = CAS.getReference(Digest);
+ if (!StoredID)
+ return reportError(StoredID.takeError(), error, true);
+
+ if (Error E = CAS.store(*StoredID, Refs, Data))
+ return reportError(std::move(E), error, true);
+ *c_id_p = llcas_objectid_t{StoredID->getOpaqueData()};
+ return false;
+}
+
+llcas_data_t llcas_loaded_object_get_data(llcas_cas_t c_cas,
+ llcas_loaded_object_t c_obj) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ ondisk::ObjectHandle Obj = ondisk::ObjectHandle(c_obj.opaque);
+ auto Data = CAS.getObjectData(Obj);
+ return llcas_data_t{Data.data(), Data.size()};
+}
+
+llcas_object_refs_t llcas_loaded_object_get_refs(llcas_cas_t c_cas,
+ llcas_loaded_object_t c_obj) {
+ auto &CAS = unwrap(c_cas)->DB->getGraphDB();
+ ondisk::ObjectHandle Obj = ondisk::ObjectHandle(c_obj.opaque);
+ auto Refs = CAS.getObjectRefs(Obj);
+ return llcas_object_refs_t{Refs.begin().getOpaqueData(),
+ Refs.end().getOpaqueData()};
+}
+
+size_t llcas_object_refs_get_count(llcas_cas_t c_cas,
+ llcas_object_refs_t c_refs) {
+ auto B = object_refs_iterator::fromOpaqueData(c_refs.opaque_b);
+ auto E = object_refs_iterator::fromOpaqueData(c_refs.opaque_e);
+ return E - B;
+}
+
+llcas_objectid_t llcas_object_refs_get_id(llcas_cas_t c_cas,
+ llcas_object_refs_t c_refs,
+ size_t index) {
+ auto RefsI = object_refs_iterator::fromOpaqueData(c_refs.opaque_b);
+ ObjectID Ref = *(RefsI + index);
+ return llcas_objectid_t{Ref.getOpaqueData()};
+}
+
+llcas_lookup_result_t
+llcas_actioncache_get_for_digest(llcas_cas_t c_cas, llcas_digest_t c_key,
+ llcas_objectid_t *p_value, bool globally,
+ char **error) {
+ auto &Wrap = *unwrap(c_cas);
+ auto &DB = *Wrap.DB;
+ ArrayRef Key(c_key.data, c_key.size);
+ std::optional<ObjectID> Value;
+ if (Error E = cacheGet(DB.getKeyValueDB(), Key).moveInto(Value))
+ return reportError(std::move(E), error, LLCAS_LOOKUP_RESULT_ERROR);
+ if (!Value) {
+ if (!globally)
+ return LLCAS_LOOKUP_RESULT_NOTFOUND;
+
+ if (Error E = Wrap.downstreamKey(Key).moveInto(Value))
+ return reportError(std::move(E), error, LLCAS_LOOKUP_RESULT_ERROR);
+ if (!Value)
+ return LLCAS_LOOKUP_RESULT_NOTFOUND;
+ }
+ *p_value = llcas_objectid_t{Value->getOpaqueData()};
+ return LLCAS_LOOKUP_RESULT_SUCCESS;
+}
+
+void llcas_actioncache_get_for_digest_async(
+ llcas_cas_t c_cas, llcas_digest_t c_key, bool globally, void *ctx_cb,
+ llcas_actioncache_get_cb cb, llcas_cancellable_t *c_cancellable) {
+ auto CancelState = std::make_shared<CancellableState>();
+ if (c_cancellable) {
+ *c_cancellable = wrap(new CancellableWrap{CancelState});
+ }
+ bool IsCancellable = c_cancellable != nullptr;
+
+ ArrayRef Key(c_key.data, c_key.size);
+ SmallVector<uint8_t, 32> KeyBuf(Key);
+
+ unwrap(c_cas)->Pool.async([=] {
+ if (IsCancellable) {
+ // Wait a bit for the caller to have a chance to cancel.
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ auto &Wrap = *unwrap(c_cas);
+ if (CancelState->Cancelled) {
+ Wrap.syncErrs([&](raw_ostream &OS) {
+ OS << "actioncache_get_for_digest_async cancelled\n";
+ });
+ return cb(ctx_cb, LLCAS_LOOKUP_RESULT_NOTFOUND, llcas_objectid_t(),
+ nullptr);
+ }
+ llcas_objectid_t c_value;
+ char *c_err;
+ llcas_lookup_result_t result = llcas_actioncache_get_for_digest(
+ c_cas, llcas_digest_t{KeyBuf.data(), KeyBuf.size()}, &c_value, globally,
+ &c_err);
+ cb(ctx_cb, result, c_value, c_err);
+ });
+}
+
+bool llcas_actioncache_put_for_digest(llcas_cas_t c_cas, llcas_digest_t c_key,
+ llcas_objectid_t c_value, bool globally,
+ char **error) {
+ auto &Wrap = *unwrap(c_cas);
+ auto &DB = *Wrap.DB;
+ ObjectID Value = ObjectID::fromOpaqueData(c_value.opaque);
+ ArrayRef Key(c_key.data, c_key.size);
+ Expected<ObjectID> Ret = cachePut(DB.getKeyValueDB(), Key, Value);
+ if (!Ret)
+ return reportError(Ret.takeError(), error, true);
+ if (*Ret != Value)
+ return reportError(
+ createStringError(errc::invalid_argument, "cache poisoned"), error,
+ true);
+
+ if (globally) {
+ if (Error E = Wrap.upstreamKey(Key, Value))
+ return reportError(std::move(E), error, true);
+ }
+
+ return false;
+}
+
+void llcas_actioncache_put_for_digest_async(
+ llcas_cas_t c_cas, llcas_digest_t c_key, llcas_objectid_t c_value,
+ bool globally, void *ctx_cb, llcas_actioncache_put_cb cb,
+ llcas_cancellable_t *c_cancellable) {
+ auto CancelState = std::make_shared<CancellableState>();
+ if (c_cancellable) {
+ *c_cancellable = wrap(new CancellableWrap{CancelState});
+ }
+ bool IsCancellable = c_cancellable != nullptr;
+
+ ArrayRef Key(c_key.data, c_key.size);
+ SmallVector<uint8_t, 32> KeyBuf(Key);
+
+ unwrap(c_cas)->Pool.async([=] {
+ if (IsCancellable) {
+ // Wait a bit for the caller to have a chance to cancel.
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ auto &Wrap = *unwrap(c_cas);
+ if (CancelState->Cancelled) {
+ Wrap.syncErrs([&](raw_ostream &OS) {
+ OS << "actioncache_put_for_digest_async cancelled\n";
+ });
+ return cb(ctx_cb, false, nullptr);
+ }
+ char *c_err;
+ bool failed = llcas_actioncache_put_for_digest(
+ c_cas, llcas_digest_t{KeyBuf.data(), KeyBuf.size()}, c_value, globally,
+ &c_err);
+ cb(ctx_cb, failed, c_err);
+ });
+}
diff --git a/llvm/tools/libCASPluginTest/libCASPluginTest.exports b/llvm/tools/libCASPluginTest/libCASPluginTest.exports
new file mode 100644
index 0000000000000..ad8bed6c6a689
--- /dev/null
+++ b/llvm/tools/libCASPluginTest/libCASPluginTest.exports
@@ -0,0 +1,31 @@
+llcas_actioncache_get_for_digest
+llcas_actioncache_get_for_digest_async
+llcas_actioncache_put_for_digest
+llcas_actioncache_put_for_digest_async
+llcas_cancellable_cancel
+llcas_cancellable_dispose
+llcas_cas_contains_object
+llcas_cas_create
+llcas_cas_dispose
+llcas_cas_get_hash_schema_name
+llcas_cas_get_objectid
+llcas_cas_get_ondisk_size
+llcas_cas_load_object
+llcas_cas_load_object_async
+llcas_cas_options_create
+llcas_cas_options_dispose
+llcas_cas_options_set_client_version
+llcas_cas_options_set_ondisk_path
+llcas_cas_options_set_option
+llcas_cas_prune_ondisk_data
+llcas_cas_set_ondisk_size_limit
+llcas_cas_store_object
+llcas_digest_parse
+llcas_digest_print
+llcas_get_plugin_version
+llcas_loaded_object_get_data
+llcas_loaded_object_get_refs
+llcas_object_refs_get_count
+llcas_object_refs_get_id
+llcas_objectid_get_digest
+llcas_string_dispose
diff --git a/llvm/unittests/CAS/ActionCacheTest.cpp b/llvm/unittests/CAS/ActionCacheTest.cpp
index b925aa4b55f66..7ac47a1530042 100644
--- a/llvm/unittests/CAS/ActionCacheTest.cpp
+++ b/llvm/unittests/CAS/ActionCacheTest.cpp
@@ -22,8 +22,8 @@ using namespace llvm::cas;
using namespace llvm::unittest::cas;
TEST_P(CASTest, ActionCacheHit) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
- std::unique_ptr<ActionCache> Cache = createActionCache();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ActionCache> Cache = createActionCache();
std::optional<ObjectProxy> ID;
ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID), Succeeded());
@@ -37,8 +37,8 @@ TEST_P(CASTest, ActionCacheHit) {
}
TEST_P(CASTest, ActionCacheMiss) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
- std::unique_ptr<ActionCache> Cache = createActionCache();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ActionCache> Cache = createActionCache();
std::optional<ObjectProxy> ID1, ID2;
ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
@@ -60,8 +60,8 @@ TEST_P(CASTest, ActionCacheMiss) {
}
TEST_P(CASTest, ActionCacheRewrite) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
- std::unique_ptr<ActionCache> Cache = createActionCache();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ActionCache> Cache = createActionCache();
std::optional<ObjectProxy> ID1, ID2;
ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
diff --git a/llvm/unittests/CAS/CASTestConfig.cpp b/llvm/unittests/CAS/CASTestConfig.cpp
index ad0812a308dc0..30e8c11f818dd 100644
--- a/llvm/unittests/CAS/CASTestConfig.cpp
+++ b/llvm/unittests/CAS/CASTestConfig.cpp
@@ -9,6 +9,8 @@
#include "CASTestConfig.h"
#include "OnDiskCommonUtils.h"
#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Config/config.h"
+#include "llvm/Support/Path.h"
#include "llvm/Support/SHA1.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"
@@ -18,6 +20,27 @@ using namespace llvm;
using namespace llvm::cas;
using namespace llvm::unittest::cas;
+// See llvm/utils/unittest/UnitTestMain/TestMain.cpp
+extern const char *TestMainArgv0;
+
+// Just a reachable symbol to ease resolving of the executable's path.
+static std::string TestStringArg1("castest-string-arg1");
+
+std::string unittest::cas::getCASPluginPath() {
+ std::string Executable =
+ sys::fs::getMainExecutable(TestMainArgv0, &TestStringArg1);
+ llvm::SmallString<256> PathBuf(sys::path::parent_path(
+ sys::path::parent_path(sys::path::parent_path(Executable))));
+#ifndef _WIN32
+ std::string LibName = "libCASPluginTest";
+ sys::path::append(PathBuf, "lib", LibName + LLVM_PLUGIN_EXT);
+#else
+ std::string LibName = "CASPluginTest";
+ sys::path::append(PathBuf, "bin", LibName + LLVM_PLUGIN_EXT);
+#endif
+ return std::string(PathBuf);
+}
+
Expected<ObjectID> CustomHasherOnDiskCASTest::store(OnDiskGraphDB &DB,
StringRef Data,
ArrayRef<ObjectID> Refs) {
@@ -106,6 +129,22 @@ INSTANTIATE_TEST_SUITE_P(SHA1, CustomHasherOnDiskCASTest,
::testing::Values(CustomHasherParam{
sha1Digest, "SHA1", sizeof(SHA1HashType)}));
+static CASTestingEnv createPlugin(int I) {
+ unittest::TempDir Temp("plugin-cas", /*Unique=*/true);
+ std::optional<
+ std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+ DBs;
+ EXPECT_THAT_ERROR(createPluginCASDatabases(getCASPluginPath(), Temp.path(),
+ /*PluginArgs=*/{})
+ .moveInto(DBs),
+ Succeeded());
+ if (!DBs)
+ return CASTestingEnv{nullptr, nullptr, std::move(Temp)};
+ return CASTestingEnv{std::move(DBs->first), std::move(DBs->second),
+ std::move(Temp)};
+}
+INSTANTIATE_TEST_SUITE_P(PluginCAS, CASTest, ::testing::Values(createPlugin));
+
#else
void unittest::cas::setMaxOnDiskCASMappingSize() {}
#endif /* LLVM_ENABLE_ONDISK_CAS */
diff --git a/llvm/unittests/CAS/CASTestConfig.h b/llvm/unittests/CAS/CASTestConfig.h
index 7b36ad9690565..68758f27c8371 100644
--- a/llvm/unittests/CAS/CASTestConfig.h
+++ b/llvm/unittests/CAS/CASTestConfig.h
@@ -37,13 +37,17 @@ class MockEnv {
};
struct CASTestingEnv {
- std::unique_ptr<llvm::cas::ObjectStore> CAS;
- std::unique_ptr<llvm::cas::ActionCache> Cache;
+ std::shared_ptr<llvm::cas::ObjectStore> CAS;
+ std::shared_ptr<llvm::cas::ActionCache> Cache;
std::optional<llvm::unittest::TempDir> Temp;
};
void setMaxOnDiskCASMappingSize();
+/// \returns the path of the libCASPluginTest dynamic library, which implements
+/// the CAS plugin API for testing purposes.
+std::string getCASPluginPath();
+
// Test fixture for on-disk data base tests.
class OnDiskCASTest : public ::testing::Test {
protected:
@@ -96,13 +100,13 @@ class CASTest
llvm::SmallVector<std::unique_ptr<llvm::unittest::cas::MockEnv>> Envs;
- std::unique_ptr<llvm::cas::ObjectStore> createObjectStore() {
+ std::shared_ptr<llvm::cas::ObjectStore> createObjectStore() {
auto TD = GetParam()(++(*NextCASIndex));
if (TD.Temp)
Dirs.push_back(std::move(*TD.Temp));
return std::move(TD.CAS);
}
- std::unique_ptr<llvm::cas::ActionCache> createActionCache() {
+ std::shared_ptr<llvm::cas::ActionCache> createActionCache() {
auto TD = GetParam()(++(*NextCASIndex));
if (TD.Temp)
Dirs.push_back(std::move(*TD.Temp));
diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt
index 7fdde1e6fb910..7f8b57ac3953a 100644
--- a/llvm/unittests/CAS/CMakeLists.txt
+++ b/llvm/unittests/CAS/CMakeLists.txt
@@ -6,6 +6,7 @@ set(ONDISK_CAS_TEST_SOURCES
OnDiskDataAllocatorTest.cpp
OnDiskKeyValueDBTest.cpp
OnDiskTrieRawHashMapTest.cpp
+ PluginCASTest.cpp
ProgramTest.cpp
UnifiedOnDiskCacheTest.cpp
)
@@ -35,3 +36,7 @@ add_llvm_unittest(CASTests
)
target_link_libraries(CASTests PRIVATE LLVMTestingSupport)
+
+if (LLVM_ENABLE_ONDISK_CAS)
+ add_dependencies(CASTests CASPluginTest)
+endif()
diff --git a/llvm/unittests/CAS/ObjectStoreTest.cpp b/llvm/unittests/CAS/ObjectStoreTest.cpp
index d49f4a2c95152..af248b6de8432 100644
--- a/llvm/unittests/CAS/ObjectStoreTest.cpp
+++ b/llvm/unittests/CAS/ObjectStoreTest.cpp
@@ -23,7 +23,7 @@ using namespace llvm::cas;
using namespace llvm::unittest::cas;
TEST_P(CASTest, PrintIDs) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
std::optional<CASID> ID1, ID2;
ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
@@ -41,7 +41,7 @@ TEST_P(CASTest, PrintIDs) {
}
TEST_P(CASTest, Blobs) {
- std::unique_ptr<ObjectStore> CAS1 = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS1 = createObjectStore();
StringRef ContentStrings[] = {
"word",
"some longer text std::string's local memory",
@@ -91,7 +91,7 @@ multiline text multiline text multiline text multiline text multiline text)",
}
// Confirm these blobs don't exist in a fresh CAS instance.
- std::unique_ptr<ObjectStore> CAS2 = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS2 = createObjectStore();
for (int I = 0, E = IDs.size(); I != E; ++I) {
std::optional<ObjectProxy> Proxy;
EXPECT_THAT_ERROR(CAS2->getProxy(IDs[I]).moveInto(Proxy), Failed());
@@ -115,7 +115,7 @@ multiline text multiline text multiline text multiline text multiline text)",
TEST_P(CASTest, BlobsBig) {
// A little bit of validation that bigger blobs are okay. Climb up to 1MB.
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
SmallString<256> String1 = StringRef("a few words");
SmallString<256> String2 = StringRef("others");
while (String1.size() < 1024U * 1024U) {
@@ -153,7 +153,7 @@ TEST_P(CASTest, BlobsBig) {
}
TEST_P(CASTest, LeafNodes) {
- std::unique_ptr<ObjectStore> CAS1 = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS1 = createObjectStore();
StringRef ContentStrings[] = {
"word",
"some longer text std::string's local memory",
@@ -211,7 +211,7 @@ multiline text multiline text multiline text multiline text multiline text)",
}
// Confirm these blobs don't exist in a fresh CAS instance.
- std::unique_ptr<ObjectStore> CAS2 = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS2 = createObjectStore();
for (int I = 0, E = IDs.size(); I != E; ++I) {
std::optional<ObjectProxy> Object;
EXPECT_THAT_ERROR(CAS2->getProxy(IDs[I]).moveInto(Object), Failed());
@@ -236,7 +236,7 @@ multiline text multiline text multiline text multiline text multiline text)",
}
TEST_P(CASTest, NodesBig) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
// Specifically check near 1MB for objects large enough they're likely to be
// stored externally in an on-disk CAS, and such that one of them will be
@@ -274,7 +274,7 @@ TEST_P(CASTest, NodesBig) {
}
TEST_P(CASTest, FileAPIs) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
auto runCommonTests =
[&CAS](function_ref<std::unique_ptr<unittest::TempFile>(char)>
@@ -377,14 +377,14 @@ static void testBlobsParallel1(ObjectStore &CAS, uint64_t BlobSize) {
}
TEST_P(CASTest, BlobsParallel) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
uint64_t Size = 1ULL * 1024;
ASSERT_NO_FATAL_FAILURE(testBlobsParallel1(*CAS, Size));
}
#ifdef EXPENSIVE_CHECKS
TEST_P(CASTest, BlobsBigParallel) {
- std::unique_ptr<ObjectStore> CAS = createObjectStore();
+ std::shared_ptr<ObjectStore> CAS = createObjectStore();
// 100k is large enough to be standalone files in our on-disk cas.
uint64_t Size = 100ULL * 1024;
ASSERT_NO_FATAL_FAILURE(testBlobsParallel1(*CAS, Size));
diff --git a/llvm/unittests/CAS/PluginCASTest.cpp b/llvm/unittests/CAS/PluginCASTest.cpp
new file mode 100644
index 0000000000000..047ac894192e2
--- /dev/null
+++ b/llvm/unittests/CAS/PluginCASTest.cpp
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Tests the plugin-backed \c ObjectStore and \c ActionCache against the mock
+/// plugin implementation in \c llvm/tools/libCASPluginTest.
+///
+//===----------------------------------------------------------------------===//
+
+#include "CASTestConfig.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Testing/Support/Error.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::unittest::cas;
+
+TEST(PluginCASTest, isMaterialized) {
+ unittest::TempDir Temp("plugin-cas", /*Unique=*/true);
+ std::string UpDir(Temp.path("up"));
+ std::string DownDir(Temp.path("down"));
+ std::pair<std::string, std::string> PluginOpts[] = {
+ {"upstream-path", std::string(UpDir)}};
+
+ {
+ std::optional<
+ std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+ DBs;
+ ASSERT_THAT_ERROR(
+ createPluginCASDatabases(getCASPluginPath(), DownDir, PluginOpts)
+ .moveInto(DBs),
+ Succeeded());
+ std::shared_ptr<ObjectStore> CAS;
+ std::shared_ptr<ActionCache> AC;
+ std::tie(CAS, AC) = std::move(*DBs);
+
+ std::optional<CASID> ID1, ID2;
+ ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
+ ASSERT_THAT_ERROR(CAS->createProxy({}, "2").moveInto(ID2), Succeeded());
+ std::optional<ObjectRef> ID2Ref = CAS->getReference(*ID2);
+ ASSERT_TRUE(ID2Ref);
+ bool IsMaterialized = false;
+ ASSERT_THAT_ERROR(CAS->isMaterialized(*ID2Ref).moveInto(IsMaterialized),
+ Succeeded());
+ EXPECT_TRUE(IsMaterialized);
+ ASSERT_THAT_ERROR(AC->put(*ID1, *ID2, /*CanBeDistributed=*/true),
+ Succeeded());
+ }
+
+ // Clear "local" cache.
+ sys::fs::remove_directories(DownDir);
+
+ {
+ std::optional<
+ std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
+ DBs;
+ ASSERT_THAT_ERROR(
+ createPluginCASDatabases(getCASPluginPath(), DownDir, PluginOpts)
+ .moveInto(DBs),
+ Succeeded());
+ std::shared_ptr<ObjectStore> CAS;
+ std::shared_ptr<ActionCache> AC;
+ std::tie(CAS, AC) = std::move(*DBs);
+
+ std::optional<CASID> ID1, ID2;
+ ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
+ ASSERT_THAT_ERROR(AC->get(*ID1, /*CanBeDistributed=*/true).moveInto(ID2),
+ Succeeded());
+ std::optional<ObjectRef> ID2Ref = CAS->getReference(*ID2);
+ ASSERT_TRUE(ID2Ref);
+ bool IsMaterialized = false;
+ ASSERT_THAT_ERROR(CAS->isMaterialized(*ID2Ref).moveInto(IsMaterialized),
+ Succeeded());
+ EXPECT_FALSE(IsMaterialized);
+
+ std::optional<ObjectProxy> Obj;
+ ASSERT_THAT_ERROR(CAS->getProxy(*ID2Ref).moveInto(Obj), Succeeded());
+ ASSERT_THAT_ERROR(CAS->isMaterialized(*ID2Ref).moveInto(IsMaterialized),
+ Succeeded());
+ EXPECT_TRUE(IsMaterialized);
+ }
+}
More information about the llvm-commits
mailing list