[llvm] 0e0f1b2 - [llvm] [Debuginfo] Debuginfod client library.
Noah Shutty via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 6 13:53:10 PST 2021
Author: Noah Shutty
Date: 2021-12-06T21:52:38Z
New Revision: 0e0f1b28fce841b077228fa23356b1ae38844ae2
URL: https://github.com/llvm/llvm-project/commit/0e0f1b28fce841b077228fa23356b1ae38844ae2
DIFF: https://github.com/llvm/llvm-project/commit/0e0f1b28fce841b077228fa23356b1ae38844ae2.diff
LOG: [llvm] [Debuginfo] Debuginfod client library.
This adds a Debuginfod client library which queries servers specified by the `DEBUGINFOD_URLS` environment variable for the debuginfo, executable, or a specified source file associated with a given build id.
Reviewed By: dblaikie
Differential Revision: https://reviews.llvm.org/D112758
Added:
llvm/include/llvm/Debuginfod/Debuginfod.h
llvm/lib/Debuginfod/CMakeLists.txt
llvm/lib/Debuginfod/Debuginfod.cpp
llvm/unittests/Debuginfod/CMakeLists.txt
llvm/unittests/Debuginfod/DebuginfodTests.cpp
Modified:
llvm/include/llvm/Support/Caching.h
llvm/lib/CMakeLists.txt
llvm/unittests/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h
new file mode 100644
index 0000000000000..fcb8ed3a9222b
--- /dev/null
+++ b/llvm/include/llvm/Debuginfod/Debuginfod.h
@@ -0,0 +1,71 @@
+//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of getCachedOrDownloadArtifact and
+/// several convenience functions for specific artifact types:
+/// getCachedOrDownloadSource, getCachedOrDownloadExecutable, and
+/// getCachedOrDownloadDebuginfo. This file also declares
+/// getDefaultDebuginfodUrls and getDefaultDebuginfodCacheDirectory.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFOD_DEBUGINFOD_H
+#define LLVM_DEBUGINFOD_DEBUGINFOD_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+
+typedef ArrayRef<uint8_t> BuildIDRef;
+
+typedef SmallVector<uint8_t, 10> BuildID;
+
+/// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS
+/// environment variable.
+Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls();
+
+/// Finds a default local file caching directory for the debuginfod client,
+/// first checking DEBUGINFOD_CACHE_PATH.
+Expected<std::string> getDefaultDebuginfodCacheDirectory();
+
+/// Finds a default timeout for debuginfod HTTP requests. Checks
+/// DEBUGINFOD_TIMEOUT environment variable, default is 90 seconds (90000 ms).
+std::chrono::milliseconds getDefaultDebuginfodTimeout();
+
+/// Fetches a specified source file by searching the default local cache
+/// directory and server URLs.
+Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
+ StringRef SourceFilePath);
+
+/// Fetches an executable by searching the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID);
+
+/// Fetches a debug binary by searching the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID);
+
+/// Fetches any debuginfod artifact using the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
+ StringRef UrlPath);
+
+/// Fetches any debuginfod artifact using the specified local cache directory,
+/// server URLs, and request timeout (in milliseconds). If the artifact is
+/// found, uses the UniqueKey for the local cache file.
+Expected<std::string> getCachedOrDownloadArtifact(
+ StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
+ ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout);
+
+} // end namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h
index 1e5fea17f7088..fbf5f3b5687ac 100644
--- a/llvm/include/llvm/Support/Caching.h
+++ b/llvm/include/llvm/Support/Caching.h
@@ -63,9 +63,10 @@ using AddBufferFn =
/// the cache directory if it does not already exist. The cache name appears in
/// error messages for errors during caching. The temporary file prefix is used
/// in the temporary file naming scheme used when writing files atomically.
-Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
- Twine CacheDirectoryPathRef,
- AddBufferFn AddBuffer);
+Expected<FileCache> localCache(
+ Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef,
+ AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
+ });
} // namespace llvm
#endif
diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt
index d88bb157a03b2..68236c34bf614 100644
--- a/llvm/lib/CMakeLists.txt
+++ b/llvm/lib/CMakeLists.txt
@@ -25,6 +25,7 @@ add_subdirectory(Object)
add_subdirectory(ObjectYAML)
add_subdirectory(Option)
add_subdirectory(Remarks)
+add_subdirectory(Debuginfod)
add_subdirectory(DebugInfo)
add_subdirectory(DWP)
add_subdirectory(ExecutionEngine)
diff --git a/llvm/lib/Debuginfod/CMakeLists.txt b/llvm/lib/Debuginfod/CMakeLists.txt
new file mode 100644
index 0000000000000..96dc8c33cc4bc
--- /dev/null
+++ b/llvm/lib/Debuginfod/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_llvm_component_library(LLVMDebuginfod
+ Debuginfod.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod
+
+ LINK_COMPONENTS
+ Support
+ )
diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp
new file mode 100644
index 0000000000000..b798c165e331f
--- /dev/null
+++ b/llvm/lib/Debuginfod/Debuginfod.cpp
@@ -0,0 +1,176 @@
+//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// This file defines the fetchInfo function, which retrieves
+/// any of the three supported artifact types: (executable, debuginfo, source
+/// file) associated with a build-id from debuginfod servers. If a source file
+/// is to be fetched, its absolute path must be specified in the Description
+/// argument to fetchInfo.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/HTTPClient.h"
+#include "llvm/Support/xxhash.h"
+
+namespace llvm {
+static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); }
+
+// Returns a binary BuildID as a normalized hex string.
+// Uses lowercase for compatibility with common debuginfod servers.
+static std::string buildIDToString(BuildIDRef ID) {
+ return llvm::toHex(ID, /*LowerCase=*/true);
+}
+
+Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
+ const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
+ if (DebuginfodUrlsEnv == NULL)
+ return SmallVector<StringRef>();
+
+ SmallVector<StringRef> DebuginfodUrls;
+ StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " ");
+ return DebuginfodUrls;
+}
+
+Expected<std::string> getDefaultDebuginfodCacheDirectory() {
+ if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
+ return CacheDirectoryEnv;
+
+ SmallString<64> CacheDirectory;
+ if (!sys::path::cache_directory(CacheDirectory))
+ return createStringError(
+ errc::io_error, "Unable to determine appropriate cache directory.");
+ return std::string(CacheDirectory);
+}
+
+std::chrono::milliseconds getDefaultDebuginfodTimeout() {
+ long Timeout;
+ const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
+ if (DebuginfodTimeoutEnv &&
+ to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
+ return std::chrono::milliseconds(Timeout * 1000);
+
+ return std::chrono::milliseconds(90 * 1000);
+}
+
+/// The following functions fetch a debuginfod artifact to a file in a local
+/// cache and return the cached file path. They first search the local cache,
+/// followed by the debuginfod servers.
+
+Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
+ StringRef SourceFilePath) {
+ SmallString<64> UrlPath;
+ sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+ buildIDToString(ID), "source",
+ sys::path::convert_to_slash(SourceFilePath));
+ return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
+ SmallString<64> UrlPath;
+ sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+ buildIDToString(ID), "executable");
+ return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
+ SmallString<64> UrlPath;
+ sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+ buildIDToString(ID), "debuginfo");
+ return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+// General fetching function.
+Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
+ StringRef UrlPath) {
+ SmallString<10> CacheDir;
+
+ Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
+ if (!CacheDirOrErr)
+ return CacheDirOrErr.takeError();
+ CacheDir = *CacheDirOrErr;
+
+ Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
+ getDefaultDebuginfodUrls();
+ if (!DebuginfodUrlsOrErr)
+ return DebuginfodUrlsOrErr.takeError();
+ SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
+ return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
+ DebuginfodUrls,
+ getDefaultDebuginfodTimeout());
+}
+
+Expected<std::string> getCachedOrDownloadArtifact(
+ StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
+ ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
+ SmallString<64> AbsCachedArtifactPath;
+ sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
+ "llvmcache-" + UniqueKey);
+
+ Expected<FileCache> CacheOrErr =
+ localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
+ if (!CacheOrErr)
+ return CacheOrErr.takeError();
+
+ FileCache Cache = *CacheOrErr;
+ // We choose an arbitrary Task parameter as we do not make use of it.
+ unsigned Task = 0;
+ Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey);
+ if (!CacheAddStreamOrErr)
+ return CacheAddStreamOrErr.takeError();
+ AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
+ if (!CacheAddStream)
+ return std::string(AbsCachedArtifactPath);
+ // The artifact was not found in the local cache, query the debuginfod
+ // servers.
+ if (!HTTPClient::isAvailable())
+ return createStringError(errc::io_error,
+ "No working HTTP client is available.");
+
+ HTTPClient Client;
+ Client.setTimeout(Timeout);
+ for (StringRef ServerUrl : DebuginfodUrls) {
+ SmallString<64> ArtifactUrl;
+ sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);
+
+ Expected<HTTPResponseBuffer> ResponseOrErr = Client.get(ArtifactUrl);
+ if (!ResponseOrErr)
+ return ResponseOrErr.takeError();
+
+ HTTPResponseBuffer &Response = *ResponseOrErr;
+ if (Response.Code != 200)
+ continue;
+
+ // We have retrieved the artifact from this server, and now add it to the
+ // file cache.
+ Expected<std::unique_ptr<CachedFileStream>> FileStreamOrErr =
+ CacheAddStream(Task);
+ if (FileStreamOrErr)
+ return FileStreamOrErr.takeError();
+ std::unique_ptr<CachedFileStream> &FileStream = *FileStreamOrErr;
+ if (!Response.Body)
+ return createStringError(
+ errc::io_error, "Unallocated MemoryBuffer in HTTPResponseBuffer.");
+
+ *FileStream->OS << StringRef(Response.Body->getBufferStart(),
+ Response.Body->getBufferSize());
+
+ // Return the path to the artifact on disk.
+ return std::string(AbsCachedArtifactPath);
+ }
+
+ return createStringError(errc::argument_out_of_domain, "build id not found");
+}
+} // namespace llvm
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 7b3804fb93758..6f4df5b585bf6 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -22,6 +22,7 @@ add_subdirectory(Bitcode)
add_subdirectory(Bitstream)
add_subdirectory(CodeGen)
add_subdirectory(DebugInfo)
+add_subdirectory(Debuginfod)
add_subdirectory(Demangle)
add_subdirectory(ExecutionEngine)
add_subdirectory(FileCheck)
diff --git a/llvm/unittests/Debuginfod/CMakeLists.txt b/llvm/unittests/Debuginfod/CMakeLists.txt
new file mode 100644
index 0000000000000..439a48a448528
--- /dev/null
+++ b/llvm/unittests/Debuginfod/CMakeLists.txt
@@ -0,0 +1,9 @@
+set(LLVM_LINK_COMPONENTS
+ Debuginfod
+ )
+
+add_llvm_unittest(DebuginfodTests
+ DebuginfodTests.cpp
+ )
+
+target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/Debuginfod/DebuginfodTests.cpp b/llvm/unittests/Debuginfod/DebuginfodTests.cpp
new file mode 100644
index 0000000000000..5e58e3005b720
--- /dev/null
+++ b/llvm/unittests/Debuginfod/DebuginfodTests.cpp
@@ -0,0 +1,49 @@
+//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/HTTPClient.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+#ifdef _WIN32
+#define setenv(name, var, ignore) _putenv_s(name, var)
+#endif
+
+using namespace llvm;
+
+// Check that the Debuginfod client can find locally cached artifacts.
+TEST(DebuginfodClient, CacheHit) {
+ int FD;
+ SmallString<64> CachedFilePath;
+ sys::fs::createTemporaryFile("llvmcache-key", "temp", FD, CachedFilePath);
+ StringRef CacheDir = sys::path::parent_path(CachedFilePath);
+ StringRef UniqueKey = sys::path::filename(CachedFilePath);
+ EXPECT_TRUE(UniqueKey.consume_front("llvmcache-"));
+ raw_fd_ostream OF(FD, true, /*unbuffered=*/true);
+ OF << "contents\n";
+ OF << CacheDir << "\n";
+ OF.close();
+ Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
+ UniqueKey, /*UrlPath=*/"/null", CacheDir,
+ /*DebuginfodUrls=*/{}, /*Timeout=*/std::chrono::milliseconds(1));
+ EXPECT_THAT_EXPECTED(PathOrErr, HasValue(CachedFilePath));
+}
+
+// Check that the Debuginfod client returns an Error when it fails to find an
+// artifact.
+TEST(DebuginfodClient, CacheMiss) {
+ // Ensure there are no urls to guarantee a cache miss.
+ setenv("DEBUGINFOD_URLS", "", /*replace=*/1);
+ HTTPClient::initialize();
+ Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
+ /*UniqueKey=*/"nonexistent-key", /*UrlPath=*/"/null");
+ EXPECT_THAT_EXPECTED(PathOrErr, Failed<StringError>());
+}
More information about the llvm-commits
mailing list