[llvm] 36f0190 - [llvm] [Debuginfod] LLVM debuginfod server.
Noah Shutty via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 7 11:33:33 PDT 2022
Author: Noah Shutty
Date: 2022-07-07T18:33:27Z
New Revision: 36f01909a0e29c1014301ed6835687a84bf0e9fa
URL: https://github.com/llvm/llvm-project/commit/36f01909a0e29c1014301ed6835687a84bf0e9fa
DIFF: https://github.com/llvm/llvm-project/commit/36f01909a0e29c1014301ed6835687a84bf0e9fa.diff
LOG: [llvm] [Debuginfod] LLVM debuginfod server.
This implements a debuginfod server in llvm using the `DebuginfodCollection` and `DebuginfodServer` classes. This is tested with lit tests against the debuginfod-find client.
The server scans 0 or more local directories for artifacts. It serves the debuginfod protocol over HTTP. Only the `executable` and `debuginfo` endpoints are supported (no `/source` endpoint).
The server also uses the debuginfod client as a fallback, so it can hit the local debuginfod cache or federate to other known debuginfod servers.
The client behavior is controllable through the standard environment variables (`DEBUGINFOD_URLS`, `DEBUGINFOD_CACHE_PATH`, `DEBUGINFOD_TIMEOUT`)
The server implements on-demand collection updates as follows:
If the build-id is not found by a local lookup, rescan immediately and look up the build-id again before returning 404. To protect against DoS attacks, do not rescan more frequently than once per N seconds (specified by `-m`).
Lit tests are provided which test the `llvm-debuginfod-find` client against the `llvm-debuginfod` server.
Reviewed By: mysterymath
Differential Revision: https://reviews.llvm.org/D114846
Added:
llvm/test/tools/llvm-debuginfod/Inputs/main-debug.exe
llvm/test/tools/llvm-debuginfod/Inputs/main.exe
llvm/test/tools/llvm-debuginfod/llvm-debuginfod.test
llvm/tools/llvm-debuginfod/CMakeLists.txt
llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp
Modified:
llvm/test/CMakeLists.txt
llvm/test/lit.cfg.py
llvm/test/lit.site.cfg.py.in
Removed:
################################################################################
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index bc2869dc536b5..b94681f54b7a7 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -6,6 +6,7 @@ llvm_canonicalize_cmake_booleans(
LLVM_ENABLE_FFI
LLVM_ENABLE_THREADS
LLVM_ENABLE_CURL
+ LLVM_ENABLE_HTTPLIB
LLVM_ENABLE_ZLIB
LLVM_ENABLE_LIBXML2
LLVM_INCLUDE_GO_TESTS
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index fe1a37ae1d012..97079af853a56 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -159,7 +159,7 @@ def get_asan_rtlib():
tools.extend([
'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
'llvm-addr2line', 'llvm-bcanalyzer', 'llvm-bitcode-strip', 'llvm-config',
- 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find',
+ 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find', 'llvm-debuginfod',
'llvm-
diff ', 'llvm-dis', 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis',
'llvm-extract', 'llvm-isel-fuzzer', 'llvm-ifs',
'llvm-install-name-tool', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib',
@@ -478,4 +478,3 @@ def exclude_unsupported_files_for_aix(dirname):
if 'aix' in config.target_triple:
for directory in ('/CodeGen/X86', '/DebugInfo', '/DebugInfo/X86', '/DebugInfo/Generic', '/LTO/X86', '/Linker'):
exclude_unsupported_files_for_aix(config.test_source_root + directory)
-
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index c3f6cb6dd3ca1..520a54bc108f5 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -40,6 +40,7 @@ config.have_zlib = @LLVM_ENABLE_ZLIB@
config.have_libxar = @LLVM_HAVE_LIBXAR@
config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
config.have_curl = @LLVM_ENABLE_CURL@
+config.have_httplib = @LLVM_ENABLE_HTTPLIB@
config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
config.enable_ffi = @LLVM_ENABLE_FFI@
config.build_examples = @LLVM_BUILD_EXAMPLES@
diff --git a/llvm/test/tools/llvm-debuginfod/Inputs/main-debug.exe b/llvm/test/tools/llvm-debuginfod/Inputs/main-debug.exe
new file mode 100755
index 0000000000000..fafb47ea95470
Binary files /dev/null and b/llvm/test/tools/llvm-debuginfod/Inputs/main-debug.exe
diff er
diff --git a/llvm/test/tools/llvm-debuginfod/Inputs/main.exe b/llvm/test/tools/llvm-debuginfod/Inputs/main.exe
new file mode 100755
index 0000000000000..9e6e5a7b316fc
Binary files /dev/null and b/llvm/test/tools/llvm-debuginfod/Inputs/main.exe
diff er
diff --git a/llvm/test/tools/llvm-debuginfod/llvm-debuginfod.test b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod.test
new file mode 100644
index 0000000000000..0aa0f968fa70c
--- /dev/null
+++ b/llvm/test/tools/llvm-debuginfod/llvm-debuginfod.test
@@ -0,0 +1,108 @@
+# REQUIRES: curl, httplib, thread_support
+
+#int main () {
+# int x = 1;
+# return x;
+#}
+#
+#Build as : clang -g main.c -o main-debug.exe
+#Then run : cp main-debug.exe main.exe && strip main.exe
+#resulting buildid: 2c39b7557c50162aaeb5a3148c9f76e6e46012e3
+
+# RUN: rm -rf %t
+# RUN: mkdir %t
+# # Query the debuginfod server for artifacts
+# RUN: DEBUGINFOD_CACHE_PATH=%t %python %s --server-cmd 'llvm-debuginfod -v -c 3 %S/Inputs' \
+# RUN: --tool-cmd 'llvm-debuginfod-find --dump --executable 2c39b7557c50162aaeb5a3148c9f76e6e46012e3' | \
+# RUN:
diff - %S/Inputs/main.exe
+# RUN: DEBUGINFOD_CACHE_PATH=%t %python %s --server-cmd 'llvm-debuginfod -v -c 3 %S/Inputs' \
+# RUN: --tool-cmd 'llvm-debuginfod-find --dump --debuginfo 2c39b7557c50162aaeb5a3148c9f76e6e46012e3' | \
+# RUN:
diff - %S/Inputs/main-debug.exe
+# Debuginfod server does not yet support source files
+
+# # The artifacts should still be present in the cache without needing to query
+# # the server.
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump \
+# RUN: --executable 2c39b7557c50162aaeb5a3148c9f76e6e46012e3 | \
+# RUN:
diff - %S/Inputs/main.exe
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump \
+# RUN: --debuginfo 2c39b7557c50162aaeb5a3148c9f76e6e46012e3 | \
+# RUN:
diff - %S/Inputs/main-debug.exe
+
+
+
+# This script is used to test the debuginfod client within a host tool against
+# the debuginfod server.
+# It first stands up the debuginfod server and then executes the tool.
+# This way the tool can make debuginfod HTTP requests to the debuginfod server.
+import argparse
+import threading
+import subprocess
+import sys
+import os
+import io
+
+# Starts the server and obtains the port number from the first line of stdout.
+# Waits until the server has completed one full directory scan before returning.
+def start_debuginfod_server(server_args):
+ process = subprocess.Popen(
+ server_args,
+ env=os.environ,
+ stdout=subprocess.PIPE)
+ port = -1
+ # Obtain the port.
+ stdout_reader = io.TextIOWrapper(process.stdout, encoding='ascii')
+ stdout_line = stdout_reader.readline()
+ port = int(stdout_line.split()[-1])
+ # Wait until a directory scan is completed.
+ while True:
+ stdout_line = stdout_reader.readline().strip()
+ print(stdout_line, file=sys.stderr)
+ if stdout_line == 'Updated collection':
+ break
+ return (process, port)
+
+# Starts the server with the specified args (if nonempty), then runs the tool
+# with specified args.
+# Sets the DEBUGINFOD_CACHE_PATH env var to point at the given cache_directory.
+# Sets the DEBUGINFOD_URLS env var to point at the local server.
+def test_tool(server_args, tool_args):
+ server_process = None
+ client_process = None
+ port = None
+ server_process, port = start_debuginfod_server(server_args)
+ try:
+ env = os.environ
+ if port is not None:
+ env['DEBUGINFOD_URLS'] = 'http://localhost:%s' % port
+ client_process = subprocess.Popen(
+ tool_args, env=os.environ)
+ client_code = client_process.wait()
+ if client_code != 0:
+ print('nontrivial client return code %s' % client_code, file=sys.stderr)
+ return 1
+ if server_process is not None:
+ server_process.terminate()
+ server_code = server_process.wait()
+ if server_code != -15:
+ print('nontrivial server return code %s' % server_code, file=sys.stderr)
+ return 1
+
+ finally:
+ if server_process is not None:
+ server_process.terminate()
+ if client_process is not None:
+ client_process.terminate()
+ return 0
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--server-cmd', default='', help='Command to start the server. If not present, no server is started.', type=str)
+ parser.add_argument('--tool-cmd', required=True, type=str)
+ args = parser.parse_args()
+ result = test_tool(args.server_cmd.split(),
+ args.tool_cmd.split())
+ sys.exit(result)
+
+if __name__ == '__main__':
+ main()
diff --git a/llvm/tools/llvm-debuginfod/CMakeLists.txt b/llvm/tools/llvm-debuginfod/CMakeLists.txt
new file mode 100644
index 0000000000000..b02aec56355fe
--- /dev/null
+++ b/llvm/tools/llvm-debuginfod/CMakeLists.txt
@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+ Debuginfod
+ Support
+ )
+add_llvm_tool(llvm-debuginfod
+ llvm-debuginfod.cpp
+ )
+if(LLVM_INSTALL_BINUTILS_SYMLINKS)
+ add_llvm_tool_symlink(debuginfod llvm-debuginfod)
+endif()
diff --git a/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp b/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp
new file mode 100644
index 0000000000000..aad8b2469fab0
--- /dev/null
+++ b/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp
@@ -0,0 +1,103 @@
+//===-- llvm-debuginfod.cpp - federating debuginfod server ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the llvm-debuginfod tool, which serves the debuginfod
+/// protocol over HTTP. The tool periodically scans zero or more filesystem
+/// directories for ELF binaries to serve, and federates requests for unknown
+/// build IDs to the debuginfod servers set in the DEBUGINFOD_URLS environment
+/// variable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Debuginfod/HTTPClient.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/ThreadPool.h"
+
+using namespace llvm;
+
+cl::OptionCategory DebuginfodCategory("llvm-debuginfod Options");
+
+static cl::list<std::string> ScanPaths(cl::Positional,
+ cl::desc("<Directories to scan>"),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<unsigned>
+ Port("p", cl::init(0),
+ cl::desc("Port to listen on. Set to 0 to bind to any available port."),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<std::string>
+ HostInterface("i", cl::init("0.0.0.0"),
+ cl::desc("Host interface to bind to."),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<int>
+ ScanInterval("t", cl::init(300),
+ cl::desc("Number of seconds to wait between subsequent "
+ "automated scans of the filesystem."),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<double> MinInterval(
+ "m", cl::init(10),
+ cl::desc(
+ "Minimum number of seconds to wait before an on-demand update can be "
+ "triggered by a request for a buildid which is not in the collection."),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<size_t>
+ MaxConcurrency("c", cl::init(0),
+ cl::desc("Maximum number of files to scan concurrently. If "
+ "0, use the hardware concurrency."),
+ cl::cat(DebuginfodCategory));
+
+static cl::opt<bool> VerboseLogging("v", cl::init(false),
+ cl::desc("Enable verbose logging."),
+ cl::cat(DebuginfodCategory));
+
+ExitOnError ExitOnErr;
+
+int main(int argc, char **argv) {
+ InitLLVM X(argc, argv);
+ HTTPClient::initialize();
+ cl::HideUnrelatedOptions({&DebuginfodCategory});
+ cl::ParseCommandLineOptions(argc, argv);
+
+ SmallVector<StringRef, 1> Paths;
+ for (const std::string &Path : ScanPaths)
+ Paths.push_back(Path);
+
+ ThreadPool Pool(hardware_concurrency(MaxConcurrency));
+ DebuginfodLog Log;
+ DebuginfodCollection Collection(Paths, Log, Pool, MinInterval);
+ DebuginfodServer Server(Log, Collection);
+
+ if (!Port)
+ Port = ExitOnErr(Server.Server.bind(HostInterface.c_str()));
+ else
+ ExitOnErr(Server.Server.bind(Port, HostInterface.c_str()));
+
+ Log.push("Listening on port " + Twine(Port).str());
+
+ Pool.async([&]() { ExitOnErr(Server.Server.listen()); });
+ Pool.async([&]() {
+ while (1) {
+ DebuginfodLogEntry Entry = Log.pop();
+ if (VerboseLogging) {
+ outs() << Entry.Message << "\n";
+ outs().flush();
+ }
+ }
+ });
+ if (Paths.size())
+ ExitOnErr(Collection.updateForever(std::chrono::seconds(ScanInterval)));
+ Pool.wait();
+ llvm_unreachable("The ThreadPool should never finish running its tasks.");
+}
More information about the llvm-commits
mailing list